@remnic/core 9.69.102 → 9.69.104

package/src/entity-mention-index-cache.ts

@@ -0,0 +1,443 @@
+import path from "node:path";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { raceAbort } from "./abort-error.js";
+import { entityMentionEpoch } from "./entity-mention-epoch.js";
+import { log } from "./logger.js";
+
+/**
+ * In-memory entity mention index.
+ *
+ * Scope (namespaces, dirs, secure-store key, alias map, native revision) is
+ * the cache key. Mention epoch and entity-file mutation are identity inside
+ * the slot, not part of the key:
+ *
+ * - metadata-only corpus bumps do not move the epoch, so the next recall hits
+ * - entity-file changes rebuild from entity files before the recall returns,
+ *   reusing memory snippets (no fact scan). A new entity is visible on that
+ *   recall; its memory snippets arrive with the next full reconcile. A new
+ *   canonical id schedules that reconcile. Callers share one scan. One
+ *   caller's abort stops that caller waiting and leaves the scan running
+ *   for the others
+ * - memory create/edit/delete moves the epoch. The recall serves the last
+ *   index immediately and one background reconcile runs, plus at most one
+ *   follow-up if that scan overlaps a later write. A recall during either
+ *   scan does not grant another follow-up. Further rescans wait for the
+ *   next recall. After writes quiesce and `settleEntityMentionIndex`
+ *   finishes, the index matches a full rebuild
+ * - secure-store key, alias map, and native revision stay in the key, so a
+ *   change is a miss and awaits a full rebuild. A locked store never reads
+ *   another key's plaintext. Evicting that older scope also stops its scan
+ *   from publishing the persisted index
+ * - a persisted index is readable per storage root. A process-wide cache
+ *   clear blocks every root until that root's own post-clear build publishes.
+ *   One root publishing does not reopen another root's file
+ */
+
+const MAX_SLOTS = 32;
+const SETTLE_ROUNDS = 8;
+
+export type EntityMentionIdentity = {
+  mentionEpoch: string;
+  entityMutation: string;
+};
+
+type ScopedStorage = {
+  dir: string;
+  hotCacheKeyId?(): string;
+  entityAliases?: Readonly<Record<string, string>> | null;
+  getEntityMutationVersion?(): number;
+};
+
+type Slot = {
+  scopeKey: string;
+  index: unknown;
+  identity: EntityMentionIdentity;
+  token: number;
+  rebuild: Promise<void> | null;
+  entityJob: Promise<void> | null;
+  again: boolean;
+  followUps: number;
+  currentIdentity: () => EntityMentionIdentity;
+  buildFull: (abortSignal?: AbortSignal) => Promise<unknown>;
+};
+
+const slots = new Map<string, Slot>();
+const rebuildsByScope = new Map<string, number>();
+const GENERATION_STRIDE = 1_000_000_000;
+let clearEpoch = 0;
+const localGenerationByDir = new Map<string, number>();
+const publishedGenerationByDir = new Map<string, number>();
+const persistTailByDir = new Map<string, Promise<void>>();
+
+function resolveMentionDir(dir: string): string {
+  return path.resolve(dir);
+}
+
+export function entityMentionCacheGeneration(dir: string): number {
+  const resolved = resolveMentionDir(dir);
+  return clearEpoch * GENERATION_STRIDE + (localGenerationByDir.get(resolved) ?? 0);
+}
+
+export function entityMentionPersistedIndexReadable(dir: string): boolean {
+  const resolved = resolveMentionDir(dir);
+  const local = localGenerationByDir.get(resolved) ?? 0;
+  if (clearEpoch === 0 && local === 0) return true;
+  return publishedGenerationByDir.get(resolved) === entityMentionCacheGeneration(resolved);
+}
+
+function dirsInScope(scopeKey: string): string[] {
+  const nsEnd = scopeKey.indexOf("\u001e");
+  const storageEnd = nsEnd === -1 ? -1 : scopeKey.indexOf("\u001e", nsEnd + 1);
+  if (nsEnd === -1 || storageEnd === -1) return [];
+  return scopeKey
+    .slice(nsEnd + 1, storageEnd)
+    .split("\u001f")
+    .map((part) => part.split("\u001d")[0] ?? "")
+    .filter((dir) => dir.length > 0)
+    .map((dir) => resolveMentionDir(dir));
+}
+
+function persistedMentionIndexFiles(scopeKey: string): string[] {
+  return dirsInScope(scopeKey).map((dir) => path.join(dir, "state", "entity-mention-index.json"));
+}
+
+function bumpScopeDirs(scopeKey: string): void {
+  for (const dir of dirsInScope(scopeKey)) {
+    localGenerationByDir.set(dir, (localGenerationByDir.get(dir) ?? 0) + 1);
+  }
+}
+
+function captureDirGenerations(scopeKey: string): Map<string, number> {
+  const captured = new Map<string, number>();
+  for (const dir of dirsInScope(scopeKey)) captured.set(dir, entityMentionCacheGeneration(dir));
+  return captured;
+}
+
+function generationsCurrent(captured: Map<string, number>): boolean {
+  for (const [dir, generation] of captured) {
+    if (entityMentionCacheGeneration(dir) !== generation) return false;
+  }
+  return true;
+}
+
+function publishDirGenerations(captured: Map<string, number>): void {
+  for (const [dir, generation] of captured) {
+    if (entityMentionCacheGeneration(dir) === generation) publishedGenerationByDir.set(dir, generation);
+  }
+}
+
+export async function writePersistedEntityMentionIndex(
+  dir: string,
+  generation: number,
+  content: string,
+): Promise<void> {
+  const resolved = resolveMentionDir(dir);
+  const previous = persistTailByDir.get(resolved) ?? Promise.resolve();
+  const run = previous.catch(() => undefined).then(async () => {
+    if (entityMentionCacheGeneration(resolved) !== generation) return;
+    const file = path.join(resolved, "state", "entity-mention-index.json");
+    await mkdir(path.dirname(file), { recursive: true });
+    const current = await readFile(file, "utf-8").catch(() => "");
+    if (current === content) return;
+    if (entityMentionCacheGeneration(resolved) !== generation) return;
+    await writeFile(file, content, "utf-8");
+  });
+  persistTailByDir.set(resolved, run.then(() => undefined, () => undefined));
+  await run;
+}
+
+function discardPersistedMentionIndex(scopeKey: string): void {
+  for (const file of persistedMentionIndexFiles(scopeKey)) {
+    void rm(file, { force: true }).catch(() => undefined);
+  }
+}
+
+function throwIfMentionAborted(signal?: AbortSignal): void {
+  if (!signal?.aborted) return;
+  const reason = signal.reason;
+  throw reason instanceof Error ? reason : new Error("entity mention rebuild aborted");
+}
+
+function compareStrings(left: string, right: string): number {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+function deduped(values: readonly string[]): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const value of values) {
+    const trimmed = value.trim();
+    if (trimmed.length === 0 || seen.has(trimmed)) continue;
+    seen.add(trimmed);
+    out.push(trimmed);
+  }
+  return out;
+}
+
+function sameIdentity(left: EntityMentionIdentity, right: EntityMentionIdentity): boolean {
+  return left.mentionEpoch === right.mentionEpoch && left.entityMutation === right.entityMutation;
+}
+
+function noteRebuild(scopeKey: string): void {
+  rebuildsByScope.set(scopeKey, (rebuildsByScope.get(scopeKey) ?? 0) + 1);
+}
+
+function scopeFamily(scopeKey: string): string {
+  const nsEnd = scopeKey.indexOf("\u001e");
+  const storageEnd = nsEnd === -1 ? -1 : scopeKey.indexOf("\u001e", nsEnd + 1);
+  const namespaceKey = nsEnd === -1 ? scopeKey : scopeKey.slice(0, nsEnd);
+  const storageKey = nsEnd === -1 || storageEnd === -1 ? "" : scopeKey.slice(nsEnd + 1, storageEnd);
+  const dirs = storageKey
+    .split("\u001f")
+    .map((part) => part.split("\u001d")[0] ?? "")
+    .join("\u001f");
+  return `${namespaceKey}\u001e${dirs}`;
+}
+
+function forgetScope(scopeKey: string, bumpPersisted = false): void {
+  const slot = slots.get(scopeKey);
+  if (slot) {
+    slot.token += 1;
+    slot.again = false;
+  }
+  if (bumpPersisted) bumpScopeDirs(scopeKey);
+  slots.delete(scopeKey);
+  rebuildsByScope.delete(scopeKey);
+}
+
+function evictOldest(scopeKey: string): void {
+  if (slots.size < MAX_SLOTS || slots.has(scopeKey)) return;
+  const oldest = slots.keys().next().value;
+  if (typeof oldest === "string") forgetScope(oldest);
+}
+
+function evictSameFamily(scopeKey: string): void {
+  const family = scopeFamily(scopeKey);
+  for (const key of [...slots.keys()]) {
+    if (key !== scopeKey && scopeFamily(key) === family) forgetScope(key, true);
+  }
+}
+
+function wantsSnippetReconcile(index: unknown): boolean {
+  return (
+    !!index &&
+    typeof index === "object" &&
+    (index as { pendingSnippetReconcile?: boolean }).pendingSnippetReconcile === true
+  );
+}
+
+function armRebuild(slot: Slot): void {
+  // A recall that arrives while a scan is already running must not grant
+  // another follow-up. The chain stops after the one already armed.
+  if (!slot.rebuild) slot.followUps = 1;
+  startRebuild(slot);
+}
+
+function startRebuild(slot: Slot): void {
+  // A recall that arrives during a scan is not a write. The follow-up is
+  // armed only when the scan's end identity differs from the identity it
+  // started with. Setting `again` here made every concurrent recall pay a
+  // second full corpus scan after a stable rebuild.
+  if (slot.rebuild) return;
+  const token = slot.token;
+  const captured = captureDirGenerations(slot.scopeKey);
+  noteRebuild(slot.scopeKey);
+  const started = slot.currentIdentity();
+  slot.again = false;
+  const handle: { pending: Promise<void> | null } = { pending: null };
+  handle.pending = slot
+    .buildFull(undefined)
+    .then((index) => {
+      if (slot.token !== token || !generationsCurrent(captured)) return;
+      publishDirGenerations(captured);
+      const ended = slot.currentIdentity();
+      slot.index = index;
+      if (sameIdentity(started, ended)) {
+        slot.identity = ended;
+        return;
+      }
+      // Scan overlapped later writes. Keep the start identity so the next
+      // recall stays stale and exactly one follow-up reconcile runs.
+      slot.identity = started;
+      slot.again = true;
+    })
+    .catch((err: unknown) => {
+      const message = err instanceof Error ? err.message : String(err);
+      log.warn(`entity mention index rebuild failed: ${message}`);
+    })
+    .finally(() => {
+      if (slot.rebuild === handle.pending) slot.rebuild = null;
+      // One automatic follow-up per caller-started rebuild. A store that keeps
+      // changing during every scan waits for the next recall (or settle) instead
+      // of chaining full scans with nobody waiting.
+      if (!slot.again || slot.followUps <= 0) return;
+      slot.again = false;
+      slot.followUps -= 1;
+      startRebuild(slot);
+    });
+  slot.rebuild = handle.pending;
+}
+
+export function entityMentionScopeKey(
+  namespaces: readonly string[] | undefined,
+  storages: readonly ScopedStorage[],
+  nativeRevision: string,
+): string {
+  const namespaceKey = deduped(namespaces ?? []).join("\u001f");
+  const storageKey = storages
+    .map((storage) => {
+      const aliases = Object.entries(storage.entityAliases ?? {}).sort(([left], [right]) =>
+        compareStrings(left, right),
+      );
+      return `${path.resolve(storage.dir)}\u001d${storage.hotCacheKeyId?.() ?? ""}\u001d${JSON.stringify(aliases)}`;
+    })
+    .join("\u001f");
+  return `${namespaceKey}\u001e${storageKey}\u001e${nativeRevision}`;
+}
+
+export function entityMentionIdentity(storages: readonly ScopedStorage[]): EntityMentionIdentity {
+  return {
+    mentionEpoch: storages.map((storage) => String(entityMentionEpoch.current(storage.dir))).join("\u001f"),
+    entityMutation: storages.map((storage) => String(storage.getEntityMutationVersion?.() ?? 0)).join("\u001f"),
+  };
+}
+
+export function entityMentionFullRebuildsStarted(scopeKey: string): number {
+  return rebuildsByScope.get(scopeKey) ?? 0;
+}
+
+export function entityMentionIndexScopeKeys(): string[] {
+  return [...slots.keys()];
+}
+
+export function dropEntityMentionIndexCache(scopeKey?: string): void {
+  if (scopeKey === undefined) {
+    clearEpoch += 1;
+    for (const key of [...slots.keys()]) {
+      discardPersistedMentionIndex(key);
+      forgetScope(key);
+    }
+    return;
+  }
+  discardPersistedMentionIndex(scopeKey);
+  forgetScope(scopeKey, true);
+}
+
+export async function settleEntityMentionIndex(scopeKey?: string): Promise<void> {
+  const matches = (slot: Slot) => scopeKey === undefined || slot.scopeKey === scopeKey;
+  for (let round = 0; round < SETTLE_ROUNDS; round += 1) {
+    for (const slot of slots.values()) {
+      if (!matches(slot)) continue;
+      if (!slot.rebuild && slot.again) {
+        slot.again = false;
+        startRebuild(slot);
+      }
+    }
+    const pending = [...slots.values()].filter((slot) => matches(slot) && slot.rebuild).map((slot) => slot.rebuild!);
+    if (pending.length === 0) return;
+    await Promise.all(pending);
+  }
+}
+
+export async function resolveEntityMentionIndex<T>(options: {
+  scopeKey: string;
+  currentIdentity: () => EntityMentionIdentity;
+  buildFull: (abortSignal?: AbortSignal) => Promise<T>;
+  rebuildEntities: (previous: T, abortSignal?: AbortSignal) => Promise<T>;
+  abortSignal?: AbortSignal;
+}): Promise<T> {
+  const live = options.currentIdentity();
+  const existing = slots.get(options.scopeKey);
+  if (!existing) {
+    evictSameFamily(options.scopeKey);
+    evictOldest(options.scopeKey);
+    noteRebuild(options.scopeKey);
+    const captured = captureDirGenerations(options.scopeKey);
+    const started = options.currentIdentity();
+    const index = await options.buildFull(options.abortSignal);
+    // A clear or family eviction during this first scan must not publish it.
+    if (!generationsCurrent(captured)) return index;
+    publishDirGenerations(captured);
+    const ended = options.currentIdentity();
+    evictSameFamily(options.scopeKey);
+    evictOldest(options.scopeKey);
+    const slot: Slot = {
+      scopeKey: options.scopeKey,
+      index,
+      identity: sameIdentity(started, ended) ? ended : started,
+      token: 0,
+      rebuild: null,
+      entityJob: null,
+      again: false,
+      followUps: 0,
+      currentIdentity: options.currentIdentity,
+      buildFull: options.buildFull,
+    };
+    slots.set(options.scopeKey, slot);
+    if (!sameIdentity(started, ended)) armRebuild(slot);
+    return index;
+  }
+  existing.currentIdentity = options.currentIdentity;
+  existing.buildFull = options.buildFull;
+  while (existing.identity.entityMutation !== options.currentIdentity().entityMutation) {
+    throwIfMentionAborted(options.abortSignal);
+    const pending = existing.entityJob;
+    if (pending) {
+      await raceAbort(pending, options.abortSignal, "entity mention rebuild aborted");
+      continue;
+    }
+    const liveNow = options.currentIdentity();
+    const epochBefore = existing.identity.mentionEpoch;
+    const mutationBefore = liveNow.entityMutation;
+    let release = (): void => {};
+    const gate = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    existing.entityJob = gate;
+    existing.token += 1;
+    existing.again = false;
+    const token = existing.token;
+    const captured = captureDirGenerations(existing.scopeKey);
+    // Shared scan. This caller's abort stops its wait and leaves the scan
+    // running. Publish when the scan settles, even if this caller has left.
+    // A family eviction bumps the token and the root generation; this scan
+    // must not install its result or start a follow-up that would republish.
+    const scan = options.rebuildEntities(existing.index as T, undefined);
+    void scan
+      .then(
+        (rebuilt) => {
+          if (existing.token !== token || !generationsCurrent(captured)) return;
+          const now = options.currentIdentity();
+          existing.index = rebuilt;
+          const epochStable = epochBefore === liveNow.mentionEpoch && epochBefore === now.mentionEpoch;
+          const mutationStable = mutationBefore === now.entityMutation;
+          existing.identity = {
+            entityMutation: mutationStable ? now.entityMutation : mutationBefore,
+            mentionEpoch: epochStable ? now.mentionEpoch : epochBefore,
+          };
+          if (existing.identity.mentionEpoch !== now.mentionEpoch || wantsSnippetReconcile(rebuilt)) {
+            armRebuild(existing);
+          }
+        },
+        () => {},
+      )
+      .finally(() => {
+        if (existing.entityJob === gate) existing.entityJob = null;
+        release();
+      });
+    const rebuilt = await raceAbort(scan, options.abortSignal, "entity mention rebuild aborted");
+    throwIfMentionAborted(options.abortSignal);
+    return rebuilt;
+  }
+  if (existing.identity.mentionEpoch !== live.mentionEpoch) {
+    armRebuild(existing);
+    return existing.index as T;
+  }
+  return existing.index as T;
+}
+
+entityMentionEpoch.registerReset(() => {
+  dropEntityMentionIndexCache();
+});

package/src/entity-retrieval.ts

@@ -2,7 +2,7 @@ import { resolveNamespaceCapabilities } from "./capabilities.js";
 import { renderAuthorityBoundContent } from "./recall-context-composition.js";
 import { createHash } from "node:crypto";
 import { sanitizeMemoryContent } from "./sanitize.js";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { collectNativeKnowledgeChunks, type NativeKnowledgeChunk } from "./native-knowledge.js";
 import { compareEntityTimestamps, normalizeEntityName, type StorageManager } from "./storage.js";
@@ -22,6 +22,15 @@ import {
   yieldEntityRecallScan,
   yieldEntityRecallScanEvery,
 } from "./entity-recall-cancellation.js";
+import { applyCarriedMemorySnippets } from "./entity-mention-carry.js";
+import {
+  entityMentionCacheGeneration,
+  entityMentionIdentity,
+  entityMentionPersistedIndexReadable,
+  entityMentionScopeKey,
+  resolveEntityMentionIndex,
+  writePersistedEntityMentionIndex,
+} from "./entity-mention-index-cache.js";
 const ENTITY_INDEX_VERSION = 3;
 const RECENT_TRANSCRIPT_LOOKBACK_HOURS = 24;
 const INSTRUCTION_LIKE_RE = /\b(always|never|must|should|remember to|do not|don't|process|workflow|template|checklist|instruction)\b/i;
@@ -64,6 +73,8 @@ type EntityMentionIndex = {
   updatedAt: string;
   entityStatusVersion?: number;
   entities: EntityMentionIndexEntry[];
+  /** In-memory only. A new canonical id had no snippets to carry. */
+  pendingSnippetReconcile?: boolean;
 };
 type EntityCandidate = {
   entry: EntityMentionIndexEntry;
@@ -356,13 +367,19 @@ async function readCurrentPersistedEntityIndex(
     return null;
   }
   const index = await readEntityIndexState(storage);
-  if (!index || index.entityStatusVersion !== storage.getMemoryStatusVersion()) {
+  if (!entityMentionPersistedIndexReadable(storage.dir) || !index || index.entityStatusVersion !== storage.getMemoryStatusVersion()) {
     return null;
   }
   return index;
 }
-const namespaceEntityIndexCache = new Map<string, EntityMentionIndex>();
-const MAX_NAMESPACE_ENTITY_INDEX_CACHE_ENTRIES = 32;
+type CarriedMemorySnippets = Map<string, { memorySnippets: string[]; memorySnippetOrigins?: (string | undefined)[] }>;
+
+function carriedMemorySnippets(index: EntityMentionIndex): CarriedMemorySnippets {
+  return new Map(index.entities.map((entry) => [entry.canonicalId, {
+    memorySnippets: entry.memorySnippets.slice(),
+    memorySnippetOrigins: entry.memorySnippetOrigins?.slice(),
+  }]));
+}
 
 function nativeEntityIndexRevision(chunks: NativeKnowledgeChunk[]): string {
   const projection = chunks.map((chunk) => [
@@ -376,40 +393,13 @@ function nativeEntityIndexRevision(chunks: NativeKnowledgeChunk[]): string {
   ]);
   return createHash("sha256").update(JSON.stringify(projection)).digest("hex");
 }
-function namespaceEntityIndexCacheKey(
-  storages: StorageManager[],
-  recallNamespaces: string[],
-  nativeRevision: string,
-): string {
-  const namespaceKey = uniqueStrings(recallNamespaces).join("\u001f");
-  const storageKey = storages
-    .map((scopedStorage) => {
-      const aliases = Object.entries(scopedStorage.entityAliases)
-        .sort(([left], [right]) => left.localeCompare(right));
-      return (
-        `${path.resolve(scopedStorage.dir)}@${scopedStorage.getMemoryStatusVersion()}` +
-        `:${scopedStorage.getMemoryCorpusVersion()}:${scopedStorage.getEntityMutationVersion()}` +
-        `:${scopedStorage.hotCacheKeyId()}:${JSON.stringify(aliases)}`
-      );
-    })
-    .join("\u001f");
-  return `${namespaceKey}\u001e${storageKey}\u001e${nativeRevision}`;
-}
-function rememberNamespaceEntityIndex(key: string, index: EntityMentionIndex): void {
-  if (namespaceEntityIndexCache.size >= MAX_NAMESPACE_ENTITY_INDEX_CACHE_ENTRIES) {
-    const oldestKey = namespaceEntityIndexCache.keys().next().value;
-    if (typeof oldestKey === "string") namespaceEntityIndexCache.delete(oldestKey);
-  }
-  namespaceEntityIndexCache.set(key, index);
-}
 
-async function writeEntityIndexState(storage: StorageManager, index: EntityMentionIndex): Promise<void> {
-  const statePath = entityIndexStatePath(storage);
-  await mkdir(path.dirname(statePath), { recursive: true });
-  const nextContent = JSON.stringify(index, null, 2) + "\n";
-  const currentContent = await readFile(statePath, "utf-8").catch(() => "");
-  if (currentContent === nextContent) return;
-  await writeFile(statePath, nextContent, "utf-8");
+async function writeEntityIndexState(
+  storage: StorageManager,
+  index: EntityMentionIndex,
+  persistGeneration: number,
+): Promise<void> {
+  await writePersistedEntityMentionIndex(storage.dir, persistGeneration, JSON.stringify(index, null, 2) + "\n");
 }
 
 function nativePseudoCanonicalId(chunk: NativeKnowledgeChunk): string {
@@ -483,6 +473,7 @@ async function buildEntityMentionIndex(
   nativeChunksOverride?: NativeKnowledgeChunk[],
   resolvedStorages?: StorageManager[],
   abortSignal?: AbortSignal,
+  carrySnippets?: CarriedMemorySnippets,
 ): Promise<EntityMentionIndex> {
   checkEntityRecallAbort(abortSignal);
   const storages = resolvedStorages ?? await resolveEntityIndexStorages(
@@ -493,11 +484,14 @@ async function buildEntityMentionIndex(
   );
   const shouldPersistIndex =
     storages.length === 1 && path.resolve(storages[0]!.dir) === path.resolve(storage.dir);
+  const persistGeneration = entityMentionCacheGeneration(storage.dir);
   const entityStatusVersionBefore = shouldPersistIndex ? storage.getMemoryStatusVersion() : undefined;
   const [previousIndex, entityFileSets, memorySets, nativeChunks] = await Promise.all([
     shouldPersistIndex ? readEntityIndexState(storage) : Promise.resolve(null),
     Promise.all(storages.map((scopedStorage) => scopedStorage.readAllEntityFiles({ abortSignal }))),
-    Promise.all(storages.map((scopedStorage) => scopedStorage.readAllMemories({ abortSignal }))),
+    carrySnippets
+      ? Promise.resolve([])
+      : Promise.all(storages.map((scopedStorage) => scopedStorage.readAllMemories({ abortSignal }))),
     nativeChunksOverride ? Promise.resolve(nativeChunksOverride) : readNativeChunks(config, recallNamespaces),
   ]);
   // The bulk reads now stop at their own scan boundaries (issue #2307); this
@@ -562,15 +556,20 @@ async function buildEntityMentionIndex(
   }
 
   scanned = 0;
-  for (const memory of memories) {
-    scanned += 1;
-    await yieldEntityRecallScanEvery(scanned, abortSignal);
-    const entry = typeof memory.frontmatter.entityRef === "string" ? entities.get(memory.frontmatter.entityRef) : undefined;
-    if (!entry) continue;
-    const snippet = await readMemorySnippet(memory);
-    if (entry.memorySnippets.includes(snippet)) continue;
-    entry.memorySnippets.push(snippet);
-    entry.memorySnippetOrigins = [...(entry.memorySnippetOrigins ?? []), memory.frontmatter.origin];
+  const pendingSnippetReconcile = carrySnippets
+    ? applyCarriedMemorySnippets(entities.values(), carrySnippets)
+    : false;
+  if (!carrySnippets) {
+    for (const memory of memories) {
+      scanned += 1;
+      await yieldEntityRecallScanEvery(scanned, abortSignal);
+      const entry = typeof memory.frontmatter.entityRef === "string" ? entities.get(memory.frontmatter.entityRef) : undefined;
+      if (!entry) continue;
+      const snippet = await readMemorySnippet(memory);
+      if (entry.memorySnippets.includes(snippet)) continue;
+      entry.memorySnippets.push(snippet);
+      entry.memorySnippetOrigins = [...(entry.memorySnippetOrigins ?? []), memory.frontmatter.origin];
+    }
   }
   // Each remaining phase is another synchronous pass over the whole index
   // (alias map, native-chunk merge, sort, two full serializations). Yield between
@@ -604,7 +603,7 @@ async function buildEntityMentionIndex(
   await yieldEntityRecallScan(abortSignal);
   const nextEntities = JSON.stringify(sortedEntities);
   const entityStatusVersionAfter = shouldPersistIndex ? storage.getMemoryStatusVersion() : undefined;
-  const canPersistIndex = shouldPersistIndex && entityStatusVersionBefore === entityStatusVersionAfter;
+  const canPersistIndex = shouldPersistIndex && entityStatusVersionBefore === entityStatusVersionAfter && persistGeneration === entityMentionCacheGeneration(storage.dir);
   const index: EntityMentionIndex = {
     version: ENTITY_INDEX_VERSION,
     updatedAt:
@@ -615,12 +614,10 @@ async function buildEntityMentionIndex(
     entities: sortedEntities,
   };
   if (canPersistIndex) {
-    // A build the caller has abandoned must not leave a persisted index behind:
-    // the write is I/O the timed-out recall no longer needs, and the next recall
-    // is already competing for the same disk.
     checkEntityRecallAbort(abortSignal);
-    await writeEntityIndexState(storage, index);
+    await writeEntityIndexState(storage, index, persistGeneration);
   }
+  if (pendingSnippetReconcile) index.pendingSnippetReconcile = true;
   return index;
 }
 
@@ -1081,30 +1078,34 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
       options.namespaceStorage,
     )
     : undefined;
-  const namespaceCacheKey =
-    resolvedStorages && options.recallNamespaces && nativeChunks
-      ? namespaceEntityIndexCacheKey(
-        resolvedStorages,
-        options.recallNamespaces,
-        nativeEntityIndexRevision(nativeChunks),
-      )
-      : undefined;
-  let index: EntityMentionIndex | undefined = namespaceCacheKey
-    ? namespaceEntityIndexCache.get(namespaceCacheKey)
-    : undefined;
-  checkEntityRecallAbort(options.abortSignal);
-  if (!index) {
-    index = await buildEntityMentionIndex(
+  if (!nativeChunks) {
+    nativeChunks = await readNativeChunks(options.config, options.recallNamespaces);
+  }
+  const indexStorages = resolvedStorages ?? [options.storage];
+  const index = await resolveEntityMentionIndex({
+    scopeKey: entityMentionScopeKey(options.recallNamespaces, indexStorages, nativeEntityIndexRevision(nativeChunks)),
+    currentIdentity: () => entityMentionIdentity(indexStorages),
+    abortSignal: options.abortSignal,
+    buildFull: (signal) => buildEntityMentionIndex(
       options.storage,
       options.config,
       options.recallNamespaces,
       options.namespaceStorage,
       nativeChunks,
-      resolvedStorages,
-      options.abortSignal,
-    );
-    if (namespaceCacheKey) rememberNamespaceEntityIndex(namespaceCacheKey, index);
-  }
+      indexStorages,
+      signal,
+    ),
+    rebuildEntities: (previous, signal) => buildEntityMentionIndex(
+      options.storage,
+      options.config,
+      options.recallNamespaces,
+      options.namespaceStorage,
+      nativeChunks,
+      indexStorages,
+      signal,
+      carriedMemorySnippets(previous),
+    ),
+  });
   // Candidate resolution is another synchronous pass over every entity and alias,
   // reached even on a cache hit that did no I/O at all — so yield here too, or a
   // cached corpus could still hold the loop past the section budget (issue #2291).

package/src/maintenance/rebuild-memory-projection.ts

@@ -589,7 +589,7 @@ async function loadAuthoritativeProjectionSnapshot(options: {
   const storage = assertInjectedStorageRooted("rebuildMemoryProjection", options.memoryDir, options.storage) ?? new StorageManager(options.memoryDir);
   // Force a fresh disk read — projection verify/rebuild must see the true
   // on-disk state, not a potentially stale in-process cache.
-  storage.invalidateAllMemoriesCacheForDir();
+  storage.invalidateAllMemoriesCacheForDir({ indexedText: false });
   const selected = selectProjectionMemories([
     await storage.readAllMemories(),
     await storage.readAllColdMemories(),