@remnic/core 9.69.48 → 9.69.50

package/src/directory-sidecars.test.ts

@@ -0,0 +1,268 @@
+import assert from "node:assert/strict";
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdir, readdir, readFile, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+
+import {
+  DIRECTORY_SIDECAR_BASENAME,
+  DIRECTORY_SIDECAR_MARKER,
+  findDirectorySidecarsForQuery,
+  loadDirectorySidecar,
+  runDirectorySidecarMaintenance,
+} from "./directory-sidecars.js";
+import {
+  isGenericRecallExcludedPath,
+  isSearchExcludedPath,
+} from "./orchestration/generic-recall-paths.js";
+
+function store(): string {
+  return mkdtempSync(path.join(os.tmpdir(), "dir-sidecars-"));
+}
+
+async function writeMemory(root: string, rel: string, body: string): Promise<string> {
+  const target = path.join(root, rel);
+  await mkdir(path.dirname(target), { recursive: true });
+  writeFileSync(target, `---\ncategory: fact\n---\n${body}\n`);
+  return target;
+}
+
+async function exists(target: string): Promise<boolean> {
+  try {
+    await stat(target);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+test("maintenance writes sidecars bottom-up and skips unchanged on rerun", async () => {
+  const root = store();
+  try {
+    await writeMemory(root, "facts/2026-01-02/fact-1.md", "Telescope calibration log for the north dome.");
+    await writeMemory(root, "facts/2026-01-03/fact-2.md", "Guide star acquisition failed twice.");
+    await writeMemory(root, "entities/obs-1.md", "The north dome observer rotation schedule.");
+
+    const first = await runDirectorySidecarMaintenance(root, true);
+    const dayDir = path.join(root, "facts", "2026-01-02");
+    const monthParent = path.join(root, "facts");
+    assert.ok(
+      first.written.some((p) => p === path.join(dayDir, DIRECTORY_SIDECAR_BASENAME)),
+      "day sidecar written",
+    );
+    assert.ok(
+      first.written.some((p) => p === path.join(monthParent, DIRECTORY_SIDECAR_BASENAME)),
+      "parent sidecar written after children (bottom-up)",
+    );
+    // Bottom-up proof: the day sidecar is written BEFORE the parent.
+    const dayIdx = first.written.findIndex((p) => p === path.join(dayDir, DIRECTORY_SIDECAR_BASENAME));
+    const parentIdx = first.written.findIndex(
+      (p) => p === path.join(monthParent, DIRECTORY_SIDECAR_BASENAME),
+    );
+    assert.ok(dayIdx < parentIdx, "child sidecar written before parent");
+
+    const daySidecar = await readFile(path.join(dayDir, DIRECTORY_SIDECAR_BASENAME), "utf8");
+    assert.ok(daySidecar.startsWith(DIRECTORY_SIDECAR_MARKER));
+    assert.match(daySidecar, /Telescope calibration/);
+    // The parent embeds the child directory's abstract.
+    const parentSidecar = await readFile(path.join(monthParent, DIRECTORY_SIDECAR_BASENAME), "utf8");
+    assert.match(parentSidecar, /2026-01-02/);
+
+    const second = await runDirectorySidecarMaintenance(root, true);
+    assert.deepEqual(second.written, [], "unchanged directories are skipped");
+    assert.deepEqual(second.removed, []);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("overview query answers from the sidecar without reading child memory files", async (t) => {
+  const root = store();
+  try {
+    const dayDir = path.join(root, "facts", "2026-01-02");
+    await writeMemory(root, "facts/2026-01-02/fact-1.md", "Telescope calibration log for the north dome.");
+    await writeMemory(root, "facts/2026-01-02/fact-2.md", "Mirror alignment drift measured at 3 arcmin.");
+    await writeMemory(root, "entities/crew-1.md", "Catering roster for the base kitchen.");
+    await runDirectorySidecarMaintenance(root, true);
+
+    const matches = await findDirectorySidecarsForQuery(root, "telescope calibration");
+    assert.ok(matches.length >= 1, "query finds a directory via its sidecar");
+    assert.equal(path.dirname(matches[0].dir), path.join(root, "facts"));
+    assert.match(matches[0].abstract, /Telescope/i);
+    // Catering directory must not outrank the telescope directory.
+    assert.ok(!matches.some((m) => m.dir === path.join(root, "entities")));
+
+    // Strong proof no child file content was read: make children unreadable
+    // (stat still works, so freshness holds) and query again.
+    if (process.getuid && process.getuid() === 0) {
+      t.skip("chmod proof skipped under root");
+    } else {
+      for (const name of await readdir(dayDir)) {
+        if (name.endsWith(".md") && name !== DIRECTORY_SIDECAR_BASENAME) {
+          chmodSync(path.join(dayDir, name), 0o000);
+        }
+      }
+      try {
+        const stillMatches = await findDirectorySidecarsForQuery(root, "mirror alignment");
+        assert.ok(stillMatches.length >= 1);
+        assert.match(stillMatches[0].overview, /Mirror alignment/);
+      } finally {
+        for (const name of await readdir(dayDir)) {
+          if (name.endsWith(".md") && name !== DIRECTORY_SIDECAR_BASENAME) {
+            chmodSync(path.join(dayDir, name), 0o644);
+          }
+        }
+      }
+    }
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("adding a file invalidates the sidecar and refresh refreshes it deterministically", async () => {
+  const root = store();
+  try {
+    const dayDir = path.join(root, "facts", "2026-01-02");
+    await writeMemory(root, "facts/2026-01-02/fact-1.md", "Boiler pressure nominal.");
+    await runDirectorySidecarMaintenance(root, true);
+
+    const before = await loadDirectorySidecar(dayDir, { refresh: false });
+    assert.equal(before?.fresh, true, "sidecar is fresh right after maintenance");
+
+    await writeMemory(root, "facts/2026-01-02/fact-2.md", "Coolant pump replaced with spare unit.");
+
+    const stale = await loadDirectorySidecar(dayDir, { refresh: false });
+    assert.equal(stale?.fresh, false, "adding a file marks the sidecar stale");
+
+    // Eager maintenance rewrites the stale sidecar.
+    const report = await runDirectorySidecarMaintenance(root, true);
+    assert.ok(
+      report.written.some((p) => p === path.join(dayDir, DIRECTORY_SIDECAR_BASENAME)),
+      "stale sidecar rewritten by maintenance",
+    );
+    const afterEager = await loadDirectorySidecar(dayDir, { refresh: false });
+    assert.equal(afterEager?.fresh, true);
+    assert.match(afterEager?.overview ?? "", /Coolant pump/);
+
+    // Lazy refresh also converges after a later direct write, with no maintenance run.
+    await writeMemory(root, "facts/2026-01-02/fact-3.md", "Thruster gimbal recalibrated.");
+    const lazilyRefreshed = await loadDirectorySidecar(dayDir);
+    assert.equal(lazilyRefreshed?.fresh, true);
+    assert.match(lazilyRefreshed?.overview ?? "", /Thruster gimbal/);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("deleting a child updates the parent and prunes emptied directories", async () => {
+  const root = store();
+  try {
+    const dayDir = path.join(root, "facts", "2026-01-02");
+    const fact1 = await writeMemory(root, "facts/2026-01-02/fact-1.md", "First observation run.");
+    await writeMemory(root, "facts/2026-01-02/fact-2.md", "Second observation run.");
+    await runDirectorySidecarMaintenance(root, true);
+
+    rmSync(fact1);
+    const report = await runDirectorySidecarMaintenance(root, true);
+    const daySidecar = await readFile(path.join(dayDir, DIRECTORY_SIDECAR_BASENAME), "utf8");
+    assert.doesNotMatch(daySidecar, /First observation/, "deleted child leaves the sidecar");
+    assert.match(daySidecar, /Second observation/);
+    assert.ok(
+      report.written.some((p) => p === path.join(dayDir, DIRECTORY_SIDECAR_BASENAME)),
+      "child deletion rewrites the day sidecar",
+    );
+    assert.ok(
+      report.written.some((p) => p === path.join(root, "facts", DIRECTORY_SIDECAR_BASENAME)),
+      "child deletion propagates to the parent sidecar",
+    );
+
+    for (const name of await readdir(dayDir)) {
+      if (name.endsWith(".md") && name !== DIRECTORY_SIDECAR_BASENAME) {
+        rmSync(path.join(dayDir, name));
+      }
+    }
+    const pruned = await runDirectorySidecarMaintenance(root, true);
+    assert.ok(
+      pruned.removed.some((p) => p === path.join(dayDir, DIRECTORY_SIDECAR_BASENAME)),
+      "emptied directory loses its sidecar",
+    );
+    assert.equal(await exists(path.join(dayDir, DIRECTORY_SIDECAR_BASENAME)), false);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("namespace scoping: lookup stays inside the requested namespace", async () => {
+  const root = store();
+  try {
+    await writeMemory(root, "namespaces/alpha/facts/2026-05-01/a-1.md", "Alpha wing telescope mirror audit.");
+    await writeMemory(root, "namespaces/beta/facts/2026-05-01/b-1.md", "Beta wing telescope mirror audit.");
+    const report = await runDirectorySidecarMaintenance(root, true);
+    assert.ok(report.written.some((p) => p.includes(path.join("alpha", "facts", "2026-05-01"))));
+    assert.ok(report.written.some((p) => p.includes(path.join("beta", "facts", "2026-05-01"))));
+
+    const scoped = await findDirectorySidecarsForQuery(root, "telescope mirror", { namespace: "beta" });
+    assert.ok(scoped.length >= 1);
+    const betaRoot = path.join(root, "namespaces", "beta");
+    assert.ok(
+      scoped.every((m) => {
+        const r = path.relative(betaRoot, m.dir);
+        return r !== "" && !r.startsWith("..") && !path.isAbsolute(r);
+      }),
+      "no hit may escape the requested namespace",
+    );
+    assert.match(scoped[0].abstract, /Beta wing/i);
+
+    const unscoped = await findDirectorySidecarsForQuery(root, "telescope mirror");
+    assert.equal(unscoped.length, 2, "unscoped query spans both namespaces");
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("sidecar files are excluded from generic recall but stay searchable", async () => {
+  assert.equal(
+    isGenericRecallExcludedPath(path.join("facts", "2026-01-02", DIRECTORY_SIDECAR_BASENAME)),
+    true,
+    "sidecar must never be injected as a recall memory",
+  );
+  assert.equal(
+    isGenericRecallExcludedPath(
+      path.join("namespaces", "beta", "facts", "2026-01-02", DIRECTORY_SIDECAR_BASENAME),
+    ),
+    true,
+  );
+  assert.equal(
+    isSearchExcludedPath(path.join("facts", "2026-01-02", DIRECTORY_SIDECAR_BASENAME)),
+    false,
+    "sidecars stay full-text searchable like OKF index files",
+  );
+});
+
+test("user-authored sidecar-named files are never clobbered or removed", async () => {
+  const root = store();
+  try {
+    const factsDir = path.join(root, "facts");
+    await mkdir(factsDir, { recursive: true });
+    const userFile = path.join(factsDir, DIRECTORY_SIDECAR_BASENAME);
+    writeFileSync(userFile, "# my own notes\n");
+    await writeMemory(root, "facts/fact-1.md", "Ordinary memory.");
+
+    const enabled = await runDirectorySidecarMaintenance(root, true);
+    assert.ok(
+      !enabled.written.some((p) => p === userFile),
+      "user file without the marker is not overwritten",
+    );
+    assert.equal(await readFile(userFile, "utf8"), "# my own notes\n");
+
+    const disabled = await runDirectorySidecarMaintenance(root, false);
+    assert.ok(
+      !disabled.removed.some((p) => p === userFile),
+      "disabling maintenance never deletes a user file",
+    );
+    assert.equal(await readFile(userFile, "utf8"), "# my own notes\n");
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});

package/src/directory-sidecars.ts

@@ -0,0 +1,383 @@
+/**
+ * @remnic/core — per-directory abstract/overview sidecars (issue #2977).
+ *
+ * One derived `overview.md` per memory directory (category roots and every
+ * directory beneath them, including `namespaces/<ns>/` subtrees), rendered
+ * bottom-up from existing memory content:
+ *
+ *   - the ABSTRACT is a single line (<=256 chars) composed from child titles;
+ *   - the OVERVIEW body lists one preview line per child memory and embeds
+ *     each child DIRECTORY's abstract, so a parent summarizes its subtree
+ *     without embedding full child overviews.
+ *
+ * Determinism: no LLM, no embeddings. Content is assembled with the same
+ * preview machinery the OKF index files use (`normalizeProjectionPreview`).
+ *
+ * Invalidation: each sidecar stores a content-free fingerprint of its subtree
+ * (child name|size|mtime for files, recursive fingerprints for child dirs).
+ * Any add/edit/delete anywhere below a directory changes its fingerprint, so
+ * `loadDirectorySidecar` can detect staleness with `readdir`+`stat` alone and
+ * refresh without reading child memory files first. Eager regeneration runs
+ * through {@link runDirectorySidecarMaintenance}; lazy refresh on read covers
+ * direct-write paths that bypass scheduled maintenance (Gotcha #43).
+ *
+ * Isolation: sidecars are derived artifacts, excluded from generic recall via
+ * `isGenericRecallExcludedPath` (basename match, the same tradeoff OKF makes
+ * for `index.md`/`log.md`) while staying full-text searchable. Files at the
+ * sidecar path WITHOUT the marker are user content: never overwritten, never
+ * removed.
+ */
+
+import { createHash } from "node:crypto";
+import { lstatSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
+import path from "node:path";
+
+import { writeFileAtomically } from "./maintenance/atomic-file.js";
+import { normalizeProjectionPreview } from "./memory-projection-format.js";
+import { OKF_RESERVED_BASENAMES } from "./okf/type-mapping.js";
+import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
+
+export const DIRECTORY_SIDECAR_BASENAME = "overview.md";
+export const DIRECTORY_SIDECAR_MARKER = "<!-- remnic-directory-sidecar -->";
+
+const ABSTRACT_MAX_CHARS = 256;
+const OVERVIEW_MAX_CHARS = 4000;
+const DEFAULT_QUERY_LIMIT = 5;
+
+/** Basenames never treated as child memories (ours plus OKF's reserved ones). */
+const RESERVED_CHILD_BASENAMES: Readonly<Record<string, true>> = Object.freeze({
+  ...OKF_RESERVED_BASENAMES,
+  [DIRECTORY_SIDECAR_BASENAME]: true,
+});
+
+export interface DirectorySidecar {
+  /** Absolute directory the sidecar describes. */
+  readonly dir: string;
+  /** Single-line L0 abstract (<=256 chars). */
+  readonly abstract: string;
+  /** Abstract plus the per-child preview body (<=4000 chars). */
+  readonly overview: string;
+  /** Number of memory `.md` children in this directory. */
+  readonly childCount: number;
+  /** False when the stored fingerprint no longer matches the subtree. */
+  readonly fresh: boolean;
+}
+
+export interface DirectorySidecarMatch extends DirectorySidecar {
+  readonly score: number;
+}
+
+export interface DirectorySidecarReport {
+  written: string[];
+  removed: string[];
+}
+
+interface ParsedSidecar {
+  abstract: string;
+  overview: string;
+  childCount: number;
+  fingerprint: string;
+}
+
+function listEntries(dir: string) {
+  try {
+    return readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+}
+
+/** Root check that refuses symlinks (fs-boundary rule, PRs #1938/#1999). */
+function isRealDirectory(entryPath: string): boolean {
+  try {
+    const stats = lstatSync(entryPath);
+    return !stats.isSymbolicLink() && stats.isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+function memoryChildren(dir: string): string[] {
+  // Dirent.isFile() is false for symlinks, so a linked memory never enters the corpus.
+  return listEntries(dir)
+    .filter((entry) => entry.isFile() && !RESERVED_CHILD_BASENAMES[entry.name] && entry.name.endsWith(".md"))
+    .map((entry) => entry.name)
+    .sort((a, b) => a.localeCompare(b));
+}
+
+function directoryChildren(dir: string): string[] {
+  // Dirent.isDirectory() is false for symlinks (fs-boundary rule, PRs #1938/#1999).
+  return listEntries(dir)
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort((a, b) => a.localeCompare(b));
+}
+
+/**
+ * Content-free subtree fingerprint: sha256 over sorted child descriptors.
+ * Files contribute `name|size|mtimeMs`; directories contribute their own
+ * recursive fingerprint, so any change below a directory invalidates every
+ * ancestor. Reads no file contents.
+ */
+export function directoryFingerprint(dir: string): string {
+  const hash = createHash("sha256");
+  for (const name of memoryChildren(dir)) {
+    try {
+      const stats = statSync(path.join(dir, name));
+      hash.update(`f|${name}|${stats.size}|${Math.trunc(stats.mtimeMs)}\n`);
+    } catch {
+      hash.update(`f|${name}|missing\n`);
+    }
+  }
+  for (const name of directoryChildren(dir)) {
+    hash.update(`d|${name}|${directoryFingerprint(path.join(dir, name))}\n`);
+  }
+  return hash.digest("hex");
+}
+
+function readFileOrNull(target: string): string | null {
+  try {
+    return readFileSync(target, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+function parseSidecar(content: string): ParsedSidecar | null {
+  if (!content.startsWith(DIRECTORY_SIDECAR_MARKER)) return null;
+  const fingerprint = /^fingerprint: ([0-9a-f]{64})$/m.exec(content)?.[1];
+  if (!fingerprint) return null;
+  const abstract = /^## Abstract\n([^\n]+)$/m.exec(content)?.[1] ?? "";
+  const overviewStart = content.indexOf("## Abstract");
+  const overview =
+    overviewStart === -1 ? "" : content.slice(overviewStart, overviewStart + OVERVIEW_MAX_CHARS).trim();
+  // The `* name — preview` child bullets are the rendered child count.
+  const childCount = (content.match(/^\* .+$/gm) ?? []).length;
+  return { abstract, overview, childCount, fingerprint };
+}
+
+function sidecarPath(dir: string): string {
+  return path.join(dir, DIRECTORY_SIDECAR_BASENAME);
+}
+
+function renderSidecar(dir: string, fingerprint: string, childAbstracts: Map<string, string>): string | null {
+  const children = memoryChildren(dir);
+  const childDirs = directoryChildren(dir).filter((name) => childAbstracts.has(path.join(dir, name)));
+  if (children.length === 0 && childDirs.length === 0) return null;
+
+  const previews = children.map((name) => {
+    try {
+      return normalizeProjectionPreview(readFileSync(path.join(dir, name), "utf8"), 120);
+    } catch {
+      return "";
+    }
+  });
+  const titles = children.map(
+    (name, index) => previews[index].split(" ").slice(0, 8).join(" ") || name.replace(/\.md$/, ""),
+  );
+  const abstract = normalizeProjectionPreview(
+    `${path.basename(dir)}/ — ${children.length} memories: ${titles.slice(0, 5).join(" · ")}`,
+    ABSTRACT_MAX_CHARS,
+  );
+
+  const lines = [
+    DIRECTORY_SIDECAR_MARKER,
+    `fingerprint: ${fingerprint}`,
+    `# ${path.basename(dir)}/ overview`,
+    "",
+    "## Abstract",
+    abstract,
+    "",
+    "## Children",
+  ];
+  for (const [index, name] of children.entries()) {
+    lines.push(`* ${name} — ${previews[index]}`);
+  }
+  for (const name of childDirs) {
+    lines.push(`### ${name}/`);
+    lines.push(childAbstracts.get(path.join(dir, name)) ?? "");
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+
+/** Scope roots whose category subtrees carry sidecars: the memory root plus every namespace root. */
+function scopeRoots(memoryDir: string): string[] {
+  const roots = [memoryDir];
+  const namespacesRoot = path.join(memoryDir, "namespaces");
+  for (const entry of listEntries(namespacesRoot)) {
+    if (entry.isDirectory() && isRealDirectory(path.join(namespacesRoot, entry.name))) {
+      roots.push(path.join(namespacesRoot, entry.name));
+    }
+  }
+  return roots;
+}
+
+/**
+ * Every directory that should carry a sidecar, deepest first (post-order) so
+ * parents render after their children and can embed child abstracts.
+ */
+function collectSidecarDirs(scopeRoot: string): string[] {
+  const dirs: string[] = [];
+  const walk = (dir: string, isCategoryRoot: boolean): boolean => {
+    let carries = memoryChildren(dir).length > 0;
+    for (const name of directoryChildren(dir)) {
+      carries = walk(path.join(dir, name), false) || carries;
+    }
+    // A directory that stopped carrying memories must still be visited so its
+    // orphaned sidecar can be pruned or rewritten (cache-invalidation completeness).
+    const hasSidecar = readFileOrNull(sidecarPath(dir))?.startsWith(DIRECTORY_SIDECAR_MARKER) === true;
+    if (isCategoryRoot || carries || hasSidecar) dirs.push(dir);
+    return carries;
+  };
+  for (const category of RECALL_FALLBACK_DIRS) {
+    const categoryDir = path.join(scopeRoot, category);
+    if (!isRealDirectory(categoryDir)) continue;
+    walk(categoryDir, true);
+  }
+  return dirs;
+}
+
+/**
+ * Regenerate directory sidecars incrementally. Unchanged sidecars are left
+ * untouched; emptied directories lose theirs; user files at the sidecar path
+ * (no marker) are never written or removed. With `enabled: false` every
+ * generated sidecar is removed — the OKF index-maintenance contract.
+ */
+export async function runDirectorySidecarMaintenance(
+  memoryDir: string,
+  enabled: boolean,
+): Promise<DirectorySidecarReport> {
+  const written: string[] = [];
+  const removed: string[] = [];
+  for (const scope of scopeRoots(memoryDir)) {
+    const childAbstracts = new Map<string, string>();
+    for (const dir of collectSidecarDirs(scope)) {
+      const target = sidecarPath(dir);
+      const raw = readFileOrNull(target);
+      const existing = raw !== null && raw.startsWith(DIRECTORY_SIDECAR_MARKER) ? raw : null;
+      if (!enabled) {
+        if (existing !== null) {
+          unlinkSync(target);
+          removed.push(target);
+        }
+        continue;
+      }
+      const next = renderSidecar(dir, directoryFingerprint(dir), childAbstracts);
+      if (next === null) {
+        if (existing !== null) {
+          unlinkSync(target);
+          removed.push(target);
+        }
+        continue;
+      }
+      const parsedNext = parseSidecar(next);
+      if (parsedNext) childAbstracts.set(dir, parsedNext.abstract);
+      if (raw === next) continue;
+      if (existing === null && raw !== null) continue; // user-authored file: hands off
+      await writeFileAtomically(target, next);
+      written.push(target);
+    }
+  }
+  return { written, removed };
+}
+
+/**
+ * Read a directory's sidecar for retrieval. By default a stale sidecar is
+ * refreshed in place (child directories first), which is what makes direct
+ * writes that bypass maintenance still converge. With `{ refresh: false }`
+ * the stored sidecar is returned as-is with `fresh: false` so callers can
+ * observe staleness without writes.
+ */
+export async function loadDirectorySidecar(
+  dir: string,
+  options: { refresh?: boolean } = {},
+): Promise<DirectorySidecar | null> {
+  const refresh = options.refresh !== false;
+  const target = sidecarPath(dir);
+  const stored = readFileOrNull(target);
+  const generated = stored !== null && stored.startsWith(DIRECTORY_SIDECAR_MARKER);
+  const parsed = generated ? parseSidecar(stored) : null;
+  const fingerprint = directoryFingerprint(dir);
+  if (parsed !== null && (parsed.fingerprint === fingerprint || !refresh)) {
+    return {
+      dir,
+      abstract: parsed.abstract,
+      overview: parsed.overview,
+      childCount: parsed.childCount,
+      fresh: parsed.fingerprint === fingerprint,
+    };
+  }
+  if (!refresh) return null;
+
+  // Refresh bottom-up so this directory's body embeds current child abstracts.
+  const childAbstracts = new Map<string, string>();
+  for (const name of directoryChildren(dir)) {
+    const child = await loadDirectorySidecar(path.join(dir, name));
+    if (child) childAbstracts.set(path.join(dir, name), child.abstract);
+  }
+  const next = renderSidecar(dir, directoryFingerprint(dir), childAbstracts);
+  if (next === null) {
+    if (generated) unlinkSync(target);
+    return null;
+  }
+  if (readFileOrNull(target) !== next) {
+    if (stored !== null && !generated) {
+      // User-authored file at the sidecar path: serve nothing rather than clobber it.
+      return null;
+    }
+    await writeFileAtomically(target, next);
+  }
+  const refreshed = parseSidecar(next);
+  return refreshed
+    ? {
+        dir,
+        abstract: refreshed.abstract,
+        overview: refreshed.overview,
+        childCount: refreshed.childCount,
+        fresh: true,
+      }
+    : null;
+}
+
+/**
+ * Overview-query primitive for hierarchical retrieval (layer 1 of #2977):
+ * score every sidecar's abstract (+3 per query term) and overview (+1 per
+ * term) lexically, then return the most specific matches — when both a
+ * directory and its descendant match, only the descendant is kept, matching
+ * the drill-down semantics the full retrieval mode will build on.
+ */
+export async function findDirectorySidecarsForQuery(
+  memoryDir: string,
+  query: string,
+  options: { namespace?: string; limit?: number } = {},
+): Promise<DirectorySidecarMatch[]> {
+  const scope = options.namespace ? path.join(memoryDir, "namespaces", options.namespace) : memoryDir;
+  if (!isRealDirectory(scope)) return [];
+  const terms = query
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((term) => term.length >= 2);
+
+  const matches: DirectorySidecarMatch[] = [];
+  for (const root of scopeRoots(scope)) {
+    for (const dir of collectSidecarDirs(root)) {
+      const sidecar = await loadDirectorySidecar(dir);
+      if (!sidecar) continue;
+      const abstractLower = sidecar.abstract.toLowerCase();
+      const overviewLower = sidecar.overview.toLowerCase();
+      let score = 0;
+      for (const term of terms) {
+        if (abstractLower.includes(term)) score += 3;
+        else if (overviewLower.includes(term)) score += 1;
+      }
+      if (score > 0) matches.push({ ...sidecar, score });
+    }
+  }
+  matches.sort((a, b) => b.score - a.score || a.dir.localeCompare(b.dir));
+
+  const limit = options.limit ?? DEFAULT_QUERY_LIMIT;
+  return matches
+    .filter((candidate) => !matches.some((other) => other.dir.startsWith(`${candidate.dir}${path.sep}`)))
+    .slice(0, limit);
+}