@remnic/core 10.0.0 → 10.0.2

package/src/harmonic-retrieval-index.test.ts

@@ -0,0 +1,387 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rename, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { getMemoryProjectionPath } from "./memory-projection-store.js";
+import { openBetterSqlite3 } from "./runtime/better-sqlite.js";
+import { recordAbstractionNode } from "./abstraction-nodes.js";
+import { searchHarmonicRetrieval, type HarmonicRetrievalResult } from "./harmonic-retrieval.js";
+import { rebuildMemoryProjection } from "./maintenance/rebuild-memory-projection.js";
+import { StorageManager } from "./storage.js";
+
+test("indexed harmonic projection matches the corpus scan and still sees later sources", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const active = await storage.writeMemory("fact", "active harmonic source payload", { source: "test" });
+    const archived = await storage.writeMemory("fact", "archived harmonic source payload", { source: "test" });
+    await storage.updateMemoryFrontmatter(archived.id, { status: "archived" });
+    await recordAbstractionNode({
+      memoryDir,
+      node: {
+        schemaVersion: 1,
+        nodeId: "mixed-sources",
+        recordedAt: "2026-10-12T00:00:00.000Z",
+        sessionKey: "agent:index",
+        kind: "topic",
+        abstractionLevel: "meso",
+        title: "stored title",
+        summary: "stored summary",
+        sourceMemoryIds: [active.id, archived.id, "missing-source"],
+      },
+    });
+
+    const beforeIndex = await searchHarmonicRetrieval({
+      memoryDir,
+      query: "active harmonic source payload",
+      maxResults: 5,
+      anchorsEnabled: false,
+    });
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+    const originalReadAll = StorageManager.prototype.readAllMemories;
+    let corpusScans = 0;
+    StorageManager.prototype.readAllMemories = function (this: StorageManager, ...args: unknown[]) {
+      corpusScans += 1;
+      return Reflect.apply(originalReadAll, this, args);
+    };
+    let afterIndex: HarmonicRetrievalResult[];
+    try {
+      afterIndex = await searchHarmonicRetrieval({
+        memoryDir,
+        query: "active harmonic source payload",
+        maxResults: 5,
+        anchorsEnabled: false,
+      });
+    } finally {
+      StorageManager.prototype.readAllMemories = originalReadAll;
+    }
+    assert.equal(corpusScans, 0, "an openable fresh index must not scan the hot corpus");
+    assert.deepEqual(afterIndex, beforeIndex);
+    assert.deepEqual(afterIndex.map((result) => result.node.sourceMemoryIds), [[active.id]]);
+    assert.equal(afterIndex[0]?.node.title.includes("active harmonic source payload"), true);
+
+    const late = await storage.writeMemory("fact", "late harmonic source payload", { source: "test" });
+    await recordAbstractionNode({
+      memoryDir,
+      node: {
+        schemaVersion: 1,
+        nodeId: "late-source",
+        recordedAt: "2026-10-12T01:00:00.000Z",
+        sessionKey: "agent:index",
+        kind: "topic",
+        abstractionLevel: "meso",
+        title: "late title",
+        summary: "late summary",
+        sourceMemoryIds: [late.id],
+      },
+    });
+    const lateResults = await searchHarmonicRetrieval({
+      memoryDir,
+      query: "late harmonic source payload",
+      maxResults: 5,
+      anchorsEnabled: false,
+    });
+    assert.equal(lateResults.some((result) => result.node.nodeId === "late-source" && result.node.sourceMemoryIds?.includes(late.id)), true);
+
+    const coldMemory = (await storage.readAllMemories()).find((memory) => memory.frontmatter.id === active.id);
+    if (!coldMemory) throw new Error("active source missing before cold move");
+    const coldPath = path.join(memoryDir, "cold", path.relative(memoryDir, coldMemory.path));
+    await mkdir(path.dirname(coldPath), { recursive: true });
+    await rename(coldMemory.path, coldPath);
+    storage.invalidateMemoryCachesForTiers(["hot", "cold"]);
+    const coldResults = await searchHarmonicRetrieval({
+      memoryDir,
+      query: "active harmonic source payload",
+      maxResults: 5,
+      anchorsEnabled: false,
+    });
+    assert.equal(coldResults.some((result) => result.node.sourceMemoryIds?.includes(active.id)), true);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+function setRebuiltAt(memoryDir: string, iso: string): void {
+  const db = openBetterSqlite3(getMemoryProjectionPath(memoryDir));
+  try {
+    db.prepare("UPDATE meta SET value = ? WHERE key = 'rebuiltAt'").run(iso);
+  } finally {
+    db.close();
+  }
+}
+
+async function nodeFor(memoryDir: string, nodeId: string, sourceMemoryIds: string[], title: string) {
+  await recordAbstractionNode({
+    memoryDir,
+    node: {
+      schemaVersion: 1,
+      nodeId,
+      recordedAt: "2026-10-12T00:00:00.000Z",
+      sessionKey: "agent:index",
+      kind: "topic",
+      abstractionLevel: "meso",
+      title,
+      summary: title,
+      sourceMemoryIds,
+    },
+  });
+}
+
+const search = (memoryDir: string, query: string) =>
+  searchHarmonicRetrieval({ memoryDir, query, maxResults: 5, anchorsEnabled: false });
+
+test("a source written after the rebuild stays found beyond two days, and unsafe or queue ids never resolve", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const base = await storage.writeMemory("fact", "baseline harmonic anchor", { source: "test" });
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+
+    // Written after the rebuild, then filed under a day three days back; the
+    // index was rebuilt four days ago, so that day is inside the probe window.
+    const late = await storage.writeMemory("fact", "aged unindexed zebra source", { source: "test" });
+    const lateMemory = (await storage.readAllMemories()).find((m) => m.frontmatter.id === late.id);
+    if (!lateMemory) throw new Error("late source missing");
+    const day = new Date(Date.now() - 3 * 86_400_000).toISOString().slice(0, 10);
+    const agedPath = path.join(memoryDir, "facts", day, `${late.id}.md`);
+    await mkdir(path.dirname(agedPath), { recursive: true });
+    await rename(lateMemory.path, agedPath);
+    storage.invalidateMemoryCachesForTiers(["hot", "cold"]);
+    setRebuiltAt(memoryDir, new Date(Date.now() - 4 * 86_400_000).toISOString());
+    await nodeFor(memoryDir, "aged-node", [late.id], "aged unindexed zebra source");
+    const aged = await search(memoryDir, "aged unindexed zebra source");
+    assert.equal(aged.some((r) => r.node.nodeId === "aged-node" && r.node.sourceMemoryIds?.includes(late.id)), true);
+
+    // Path traversal ids and question-queue ids are dropped, exactly as the
+    // corpus scan drops them.
+    const questionId = await storage.writeQuestion("queued zebra question", "ctx", 1);
+    await nodeFor(memoryDir, "unsafe-node", ["../facts/" + base.id, "../../etc/passwd"], "unsafe traversal cue");
+    await nodeFor(memoryDir, "question-node", [questionId], "queue leak cue");
+    setRebuiltAt(memoryDir, new Date().toISOString());
+    for (const query of ["unsafe traversal cue", "queue leak cue"]) {
+      assert.deepEqual(await search(memoryDir, query), [], query);
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("an index older than a week falls back to the corpus scan and still finds an aged source", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const source = await storage.writeMemory("fact", "stale index quokka source", { source: "test" });
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+    setRebuiltAt(memoryDir, new Date(Date.now() - 30 * 86_400_000).toISOString());
+    await nodeFor(memoryDir, "stale-node", [source.id], "stale index quokka source");
+    const results = await search(memoryDir, "stale index quokka source");
+    assert.equal(results.some((r) => r.node.nodeId === "stale-node"), true);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// Counts hot-corpus scans while `run` executes.
+async function countCorpusScans<T>(run: () => Promise<T>): Promise<{ value: T; scans: number }> {
+  const original = StorageManager.prototype.readAllMemories;
+  let scans = 0;
+  StorageManager.prototype.readAllMemories = function (this: StorageManager, ...args: unknown[]) {
+    scans += 1;
+    return Reflect.apply(original, this, args);
+  };
+  try {
+    return { value: await run(), scans };
+  } finally {
+    StorageManager.prototype.readAllMemories = original;
+  }
+}
+
+test("a scoped or pre-complete projection is not trusted and falls back to the corpus scan", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const source = await storage.writeMemory("fact", "scoped rebuild okapi source", { source: "test" });
+    await nodeFor(memoryDir, "okapi-node", [source.id], "scoped rebuild okapi source");
+
+    // First projection built with a scope that excludes the source: rebuiltAt is
+    // fresh but memory_current does not hold it.
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false, updatedBefore: "2000-01-01T00:00:00.000Z" });
+    const scoped = await countCorpusScans(() => search(memoryDir, "scoped rebuild okapi source"));
+    assert.equal(scoped.value.some((r) => r.node.nodeId === "okapi-node"), true, "out-of-scope source must still resolve");
+    assert.equal(scoped.scans > 0, true, "an incomplete index must fall back to the scan");
+
+    // An unscoped rebuild is complete and takes the index path.
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+    const full = await countCorpusScans(() => search(memoryDir, "scoped rebuild okapi source"));
+    assert.equal(full.value.some((r) => r.node.nodeId === "okapi-node"), true);
+    assert.equal(full.scans, 0);
+
+    // A projection written before the `complete` flag existed has no such key.
+    const db = openBetterSqlite3(getMemoryProjectionPath(memoryDir));
+    try {
+      db.prepare("DELETE FROM meta WHERE key = 'complete'").run();
+    } finally {
+      db.close();
+    }
+    const legacy = await countCorpusScans(() => search(memoryDir, "scoped rebuild okapi source"));
+    assert.equal(legacy.value.some((r) => r.node.nodeId === "okapi-node"), true);
+    assert.equal(legacy.scans > 0, true, "a projection without the flag must fall back");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("a legacy memory whose id contains a slash still resolves through the projection", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const slashId = "legacy/slash-id";
+    const day = new Date().toISOString().slice(0, 10);
+    await mkdir(path.join(memoryDir, "facts", day), { recursive: true });
+    await writeFile(
+      path.join(memoryDir, "facts", day, "legacy-slash-file.md"),
+      `---\nid: ${slashId}\ncategory: fact\ncreated: ${new Date().toISOString()}\nupdated: ${new Date().toISOString()}\nsource: test\nconfidence: 0.8\nconfidenceTier: implied\ntags: ["alpha"]\n---\n\nslash identifier narwhal source\n`,
+      "utf-8",
+    );
+    await nodeFor(memoryDir, "slash-node", [slashId], "slash identifier narwhal source");
+    const before = await search(memoryDir, "slash identifier narwhal source");
+    assert.equal(before.some((r) => r.node.nodeId === "slash-node"), true, "the corpus scan keys by frontmatter id");
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+    const indexed = await countCorpusScans(() => search(memoryDir, "slash identifier narwhal source"));
+    assert.deepEqual(indexed.value, before);
+    assert.equal(indexed.scans, 0);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+function memoryFileText(id: string, body: string): string {
+  const now = new Date().toISOString();
+  return `---\nid: ${id}\ncategory: fact\ncreated: ${now}\nupdated: ${now}\nsource: test\nconfidence: 0.8\nconfidenceTier: implied\ntags: ["alpha"]\n---\n\n${body}\n`;
+}
+
+test("direct probes reject symlinked and artifact sources like the scan, and an old-dated post-rebuild source is still found", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  const outsideDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-outside-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const day = new Date().toISOString().slice(0, 10);
+    await mkdir(path.join(memoryDir, "facts", day), { recursive: true });
+    await mkdir(path.join(memoryDir, "artifacts", day), { recursive: true });
+
+    // A source file outside the namespace root, reachable through a symlink.
+    await writeFile(path.join(outsideDir, "outside-sym.md"), memoryFileText("outside-sym", "symlink escape cue body"), "utf-8");
+    await symlink(path.join(outsideDir, "outside-sym.md"), path.join(memoryDir, "facts", day, "outside-sym.md"));
+    // An artifact: outside the recall surface the corpus scan reads.
+    await writeFile(path.join(memoryDir, "artifacts", day, "art-1.md"), memoryFileText("art-1", "artifact leak cue body"), "utf-8");
+    await nodeFor(memoryDir, "symlink-node", ["outside-sym"], "symlink escape cue");
+    await nodeFor(memoryDir, "artifact-node", ["art-1"], "artifact leak cue");
+
+    const scanSymlink = await search(memoryDir, "symlink escape cue");
+    const scanArtifact = await search(memoryDir, "artifact leak cue");
+    assert.deepEqual(scanSymlink, [], "the corpus scan must not follow the symlink");
+    assert.deepEqual(scanArtifact, [], "the corpus scan must not read artifacts");
+
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+    const indexed = await countCorpusScans(async () => ({
+      symlink: await search(memoryDir, "symlink escape cue"),
+      artifact: await search(memoryDir, "artifact leak cue"),
+    }));
+    assert.deepEqual(indexed.value.symlink, scanSymlink);
+    assert.deepEqual(indexed.value.artifact, scanArtifact);
+    assert.equal(indexed.scans, 0, "this was the index path, not a fallback");
+
+    // Written after the rebuild but filed under a date far outside the probe
+    // window (an import that keeps its original created date).
+    await mkdir(path.join(memoryDir, "facts", "2020-01-01"), { recursive: true });
+    await writeFile(
+      path.join(memoryDir, "facts", "2020-01-01", "imported-old.md"),
+      memoryFileText("imported-old", "imported older marmot cue body"),
+      "utf-8",
+    );
+    storage.invalidateMemoryCachesForTiers(["hot", "cold"]);
+    await nodeFor(memoryDir, "imported-node", ["imported-old"], "imported older marmot cue");
+    const old = await search(memoryDir, "imported older marmot cue");
+    assert.equal(old.some((r) => r.node.nodeId === "imported-node" && r.node.sourceMemoryIds?.includes("imported-old")), true);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+    await rm(outsideDir, { recursive: true, force: true });
+  }
+});
+
+test("a dangling source id costs one path walk, then is remembered as absent", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const real = await storage.writeMemory("fact", "real source wombat body", { source: "test" });
+    await nodeFor(memoryDir, "real-node", [real.id, "dangling-id"], "real source wombat");
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+
+    const original = StorageManager.prototype.collectActiveMemoryPaths;
+    let walks = 0;
+    StorageManager.prototype.collectActiveMemoryPaths = function (this: StorageManager, ...args: unknown[]) {
+      walks += 1;
+      return Reflect.apply(original, this, args);
+    };
+    try {
+      const first = await search(memoryDir, "real source wombat");
+      assert.equal(first.some((r) => r.node.nodeId === "real-node" && r.node.sourceMemoryIds?.includes(real.id)), true);
+      assert.equal(walks, 1, "the unresolved id is looked up by one walk");
+      await search(memoryDir, "real source wombat");
+      assert.equal(walks, 1, "a confirmed-absent id is not walked again within the TTL");
+    } finally {
+      StorageManager.prototype.collectActiveMemoryPaths = original;
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+
+test("a separator-containing id written after the rebuild is still placed, by the scan", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+    const day = new Date().toISOString().slice(0, 10);
+    await mkdir(path.join(memoryDir, "facts", day), { recursive: true });
+    await writeFile(
+      path.join(memoryDir, "facts", day, "capsule-import-file.md"),
+      memoryFileText("capsule/import-id", "capsule import ocelot source body"),
+      "utf-8",
+    );
+    storage.invalidateMemoryCachesForTiers(["hot", "cold"]);
+    await nodeFor(memoryDir, "capsule-node", ["capsule/import-id"], "capsule import ocelot source");
+    const results = await search(memoryDir, "capsule import ocelot source");
+    assert.equal(results.some((r) => r.node.nodeId === "capsule-node" && r.node.sourceMemoryIds?.includes("capsule/import-id")), true);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("the cold path walk honors an aborted signal", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-harmonic-index-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    await mkdir(path.join(memoryDir, "cold", "facts", "2020-01-01"), { recursive: true });
+    await writeFile(
+      path.join(memoryDir, "cold", "facts", "2020-01-01", "cold-1.md"),
+      memoryFileText("cold-1", "cold walk abort body"),
+      "utf-8",
+    );
+    assert.equal((await storage.collectColdMemoryPaths()).length, 1);
+    await assert.rejects(() => storage.collectColdMemoryPaths({ abortSignal: AbortSignal.abort() }));
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/harmonic-retrieval.ts

@@ -1,4 +1,6 @@
 import path from "node:path";
+import { existsSync } from "node:fs";
+import { realpath } from "node:fs/promises";
 import { listJsonFilesStrict, readJsonFile } from "./json-store.js";
 import { throwIfAborted } from "./abort-error.js";
 import { inferMemoryStatus, toMemoryPathRel } from "./memory-lifecycle-ledger-utils.js";
@@ -13,30 +15,168 @@ import { compareDeterministicStrings } from "./deterministic-order.js";
 import { countRecallTokenOverlap, normalizeRecallTokens } from "./recall-tokenization.js";
 import { stripAttributesSuffix } from "./structured-attributes.js";
 import { isValidityExpiredNow } from "./temporal-validity.js";
-import { isRecord } from "./store-contract.js";
+import { assertSafePathSegment, isRecord } from "./store-contract.js";
 import { isSupportPassportPrivateMemory } from "./support-passport/card-projection.js";
 import type { MemoryFile } from "./types.js";
+import { readProjectedMemoryStates } from "./memory-projection-batch.js";
+import { SecureStoreLockedError } from "./secure-store/secure-fs.js";
+import { StorageManager } from "./storage.js";
+import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
+import { isPathInsideStorageRoot } from "./storage-paths.js";
+
 
 type SourceMemoryMap = Map<string, MemoryFile>;
 
-async function readSourceMemories(options: {
+// `rootReal` set: a direct probe, which skips the corpus walk's symlink
+// hardening, so the resolved file must stay inside the memory root. Null: the
+// path came from the contained corpus walk and is already trusted.
+async function readMemoryFile(
+  storage: StorageManager,
+  rootReal: string | null,
+  filePath: string,
+  abortSignal?: AbortSignal,
+): Promise<MemoryFile | null> {
+  throwIfAborted(abortSignal, "harmonic retrieval aborted");
+  if (!existsSync(filePath)) return null;
+  try {
+    if (rootReal !== null && !isPathInsideStorageRoot(rootReal, await realpath(filePath))) return null;
+    return await storage.readMemoryByPath(filePath);
+  } catch (err) {
+    throwIfAborted(abortSignal, "harmonic retrieval aborted");
+    if (err instanceof SecureStoreLockedError) throw err;
+    return null;
+  }
+}
+
+// An index older than this is treated as unusable: the date window needed to
+// cover "created since the last rebuild" would stop being cheap.
+const INDEX_MAX_AGE_MS = 7 * 86_400_000;
+const DAY_MS = 86_400_000;
+
+function utcDaysSince(rebuiltMs: number, nowMs: number): string[] {
+  const days: string[] = [];
+  for (let day = Math.floor((rebuiltMs - DAY_MS) / DAY_MS) * DAY_MS; day <= nowMs; day += DAY_MS) {
+    days.push(new Date(day).toISOString().slice(0, 10));
+  }
+  return days;
+}
+
+// Source ids confirmed absent by a corpus path walk. A node only references a
+// source after the source file exists, so a confirmed absence stays true unless
+// the file is restored later; the short TTL bounds that. Without this, one
+// dangling id (a hard-deleted fact) would force a walk on every recall.
+const ABSENT_SOURCE_TTL_MS = 10 * 60_000;
+const ABSENT_SOURCE_MAX_ENTRIES = 4096;
+const absentSourceIds = new Map<string, number>(); // `${memoryDir}\0${id}` -> expiry ms
+
+// Memories the projection has no row for were written after its last rebuild, so
+// their dated directory is normally one of `days`. Hot and cold tiers are probed
+// by exact path. Only recall-corpus categories are probed: `questions/` is a queue
+// and `artifacts/` is outside the recall surface, so neither can be a source.
+function unprojectedCandidatePaths(memoryDir: string, id: string, days: string[]): string[] {
+  const paths: string[] = [];
+  for (const root of [memoryDir, path.join(memoryDir, "cold")]) {
+    for (const dir of RECALL_FALLBACK_DIRS) {
+      paths.push(path.join(root, dir, `${id}.md`));
+      for (const day of days) paths.push(path.join(root, dir, day, `${id}.md`));
+    }
+  }
+  return paths;
+}
+
+/**
+ * Resolve only the source ids the abstraction nodes reference. Returns null when
+ * the projection is absent, unreadable, too old, or not known to cover every
+ * active memory (a scoped or pre-`complete` rebuild), so the caller keeps the
+ * full-corpus scan. Every file is read from disk, so status, tags and validity
+ * are current; the index only says where to look. An id the cheap probes cannot
+ * place (an import filed under an old date, a tier root outside the memory dir)
+ * is looked up by one contained path-only walk, never a frontmatter parse of the
+ * corpus.
+ */
+async function readIndexedSourceMemories(options: {
+  memoryDir: string;
+  abortSignal?: AbortSignal;
+  sourceMemoryIds: string[];
+  nowMs: number;
+}): Promise<SourceMemoryMap | null> {
+  // The ids go to SQL as bound parameters, so any id the projection holds is safe to look up.
+  const batch = readProjectedMemoryStates(options.memoryDir, options.sourceMemoryIds);
+  const rebuiltMs = Date.parse(batch?.rebuiltAt ?? "");
+  if (!batch?.complete || !Number.isFinite(rebuiltMs) || options.nowMs - rebuiltMs > INDEX_MAX_AGE_MS) return null;
+  const rootReal = await realpath(options.memoryDir).catch(() => null);
+  if (rootReal === null) return null;
+
+  const storage = new StorageManager(options.memoryDir);
+  const days = utcDaysSince(rebuiltMs, options.nowMs);
+  const sourceMemories: SourceMemoryMap = new Map();
+  const pending: string[] = [];
+  for (const id of options.sourceMemoryIds) {
+    const state = batch.states.get(id);
+    // An id is only ever joined into a path when it is a single safe segment.
+    let safeId = true;
+    try {
+      assertSafePathSegment(id, "sourceMemoryId");
+    } catch {
+      safeId = false;
+    }
+    // An unprojected id that is not a single path segment cannot be probed or
+    // matched by filename, and the scan keys sources by frontmatter id, so only
+    // the scan can place it. Rare (legacy/imported ids); worst case is the old cost.
+    if (!state && !safeId) return null;
+    // state.path was containment-checked by the projection reader.
+    const candidates = state
+      ? [state.path, path.join(options.memoryDir, "cold", state.pathRel)]
+      : unprojectedCandidatePaths(options.memoryDir, id, days);
+    let found = false;
+    for (const candidate of candidates) {
+      const memory = await readMemoryFile(storage, rootReal, candidate, options.abortSignal);
+      if (memory?.frontmatter.id !== id) continue;
+      sourceMemories.set(id, memory);
+      found = true;
+      break;
+    }
+    const absentKey = `${options.memoryDir}\0${id}`;
+    if (!found && safeId && (absentSourceIds.get(absentKey) ?? 0) <= options.nowMs) pending.push(id);
+  }
+  if (pending.length === 0) return sourceMemories;
+
+  const wanted = new Set(pending);
+  const walked = [
+    ...(await storage.collectActiveMemoryPaths({ abortSignal: options.abortSignal })),
+    ...(await storage.collectColdMemoryPaths({ abortSignal: options.abortSignal })),
+  ];
+  for (const filePath of walked) {
+    throwIfAborted(options.abortSignal, "harmonic retrieval aborted");
+    if (!wanted.has(path.basename(filePath, ".md"))) continue;
+    const memory = await readMemoryFile(storage, null, filePath, options.abortSignal);
+    if (memory && wanted.has(memory.frontmatter.id)) sourceMemories.set(memory.frontmatter.id, memory);
+  }
+  if (absentSourceIds.size > ABSENT_SOURCE_MAX_ENTRIES) absentSourceIds.clear();
+  for (const id of pending) {
+    if (!sourceMemories.has(id)) absentSourceIds.set(`${options.memoryDir}\0${id}`, options.nowMs + ABSENT_SOURCE_TTL_MS);
+  }
+  return sourceMemories;
+}
+
+async function readCorpusSourceMemories(options: {
   memoryDir: string;
   abortSignal?: AbortSignal;
 }): Promise<SourceMemoryMap> {
   const sourceMemories: SourceMemoryMap = new Map();
- try {
- const { StorageManager } = await import("./storage.js");
- const storage = new StorageManager(options.memoryDir);
- const [hotMemories, coldMemories] = await Promise.all([
- storage.readAllMemories({ abortSignal: options.abortSignal }),
- storage.readAllColdMemories(),
- ]);
- for (const memory of [...hotMemories, ...coldMemories]) {
+  try {
+    const storage = new StorageManager(options.memoryDir);
+    const [hotMemories, coldMemories] = await Promise.all([
+      storage.readAllMemories({ abortSignal: options.abortSignal }),
+      storage.readAllColdMemories(),
+    ]);
+    for (const memory of [...hotMemories, ...coldMemories]) {
       throwIfAborted(options.abortSignal, "harmonic retrieval aborted");
       sourceMemories.set(memory.frontmatter.id, memory);
     }
-  } catch {
+  } catch (err) {
     throwIfAborted(options.abortSignal, "harmonic retrieval aborted");
+    if (err instanceof SecureStoreLockedError) return new Map();
     // Source validation fails closed when the authorized memory directory is
     // unreadable. Source-less nodes remain eligible for retrieval.
     sourceMemories.clear();
@@ -44,6 +184,27 @@ async function readSourceMemories(options: {
   return sourceMemories;
 }
 
+async function readSourceMemories(options: {
+  memoryDir: string;
+  abortSignal?: AbortSignal;
+  sourceMemoryIds: string[];
+  nowMs?: number;
+}): Promise<SourceMemoryMap> {
+  try {
+    const indexed = await readIndexedSourceMemories({
+      memoryDir: options.memoryDir,
+      abortSignal: options.abortSignal,
+      sourceMemoryIds: options.sourceMemoryIds,
+      nowMs: options.nowMs ?? Date.now(),
+    });
+    if (indexed) return indexed;
+  } catch (err) {
+    throwIfAborted(options.abortSignal, "harmonic retrieval aborted");
+    if (err instanceof SecureStoreLockedError) return new Map();
+  }
+  return readCorpusSourceMemories(options);
+}
+
 function parseSourceMemoryInsertedAt(node: AbstractionNode): Map<string, string> {
   const insertedAtById = new Map<string, string>();
   const insertedAtRaw = node.metadata?.[HARMONIC_SOURCE_MEMORY_INSERTED_AT_KEY];
@@ -304,10 +465,15 @@ export async function readProjectedHarmonicGraph(options: {
 }): Promise<ProjectedHarmonicGraph> {
   throwIfAborted(options.abortSignal, "harmonic retrieval aborted");
   const nodes = await readAbstractionNodes(options);
-  const sourceBacked = nodes.some((node) => (node.sourceMemoryIds?.length ?? 0) > 0);
-  const sourceMemories: SourceMemoryMap = sourceBacked
-    ? await readSourceMemories(options)
-    : new Map<string, MemoryFile>();
+  const sourceMemoryIds = [...new Set(nodes.flatMap((node) => node.sourceMemoryIds ?? []))];
+  const sourceMemories: SourceMemoryMap = sourceMemoryIds.length > 0
+    ? await readSourceMemories({
+        memoryDir: options.memoryDir,
+        abortSignal: options.abortSignal,
+        sourceMemoryIds,
+        nowMs: options.nowMs,
+      })
+    : new Map();
   const nowMs = options.nowMs ?? Date.now();
   // A node whose full stored source set survived projection keeps legacy
   // anchor attribution (an anchor written before per-node source ids existed).

package/src/maintenance/rebuild-memory-projection.ts

@@ -755,6 +755,7 @@ function writeProjectionDb(
   governance: Awaited<ReturnType<typeof loadLatestGovernanceProjection>>,
   usedLifecycleLedger: boolean,
   sourceLifecycleLedgerEventCount: number,
+  complete: boolean,
 ): void {
   const db = openBetterSqlite3(dbPath);
   const memoryDir = path.dirname(path.dirname(dbPath));
@@ -765,6 +766,7 @@ function writeProjectionDb(
     insertMeta.run("schemaVersion", String(MEMORY_PROJECTION_SCHEMA_VERSION));
     insertMeta.run("rebuiltAt", nowIso);
     insertMeta.run("usedLifecycleLedger", usedLifecycleLedger ? "true" : "false");
+    insertMeta.run("complete", complete ? "true" : "false");
     writeProjectionMetaHighWater(insertMeta, sourceLifecycleLedgerEventCount, timelineRows);
     insertMeta.run("latestGovernanceRunId", governance?.runId ?? "");
 
@@ -1126,6 +1128,7 @@ export async function rebuildMemoryProjection(
       nextGovernance,
       snapshot.usedLifecycleLedger,
       snapshot.sourceLifecycleLedgerEventCount,
+      !hasScopedProjectionFilter(snapshot.scope),
     );
     backupPath = await backupExistingProjection(options.memoryDir, outputPath, now);
     backupPath = await commitPreparedFileAtomically(tempPath, outputPath, backupPath);