@remnic/core 9.3.755 → 9.3.757

package/src/storage.ts

@@ -4,8 +4,9 @@ import { createHash } from "node:crypto"
 import { normalizeContent, computeContentHash } from "./content-hash.js";;
 import path from "node:path";
 import { log } from "./logger.js";
-import { EntityStore } from "./storage/entity-store.js";
+import { MemoryReadStore } from "./storage/memory-read-store.js";
 import { selfDeps } from "./orchestration/self-deps.js";
+import { EntityStore } from "./storage/entity-store.js";
 import { isErrnoCode } from "./utils/errno.js";
 import { RECALL_FALLBACK_DIRS, getCategoryDir, categoryDirName } from "./utils/category-dir.js";
 import { assertPathInsideRoot } from "./utils/path-containment.js";
@@ -521,7 +522,7 @@ function parseLinkReasonValue(rawValue: string): string {
   }
 }
 
-function parseFrontmatterStringValue(rawValue: string | undefined): string | undefined {
+export function parseFrontmatterStringValue(rawValue: string | undefined): string | undefined {
   if (rawValue === undefined) return undefined;
   const trimmed = rawValue.trim();
   if (trimmed.length === 0) return undefined;
@@ -567,7 +568,7 @@ function parseReinforcementCountField(raw: string | undefined): number | undefin
   return n;
 }
 
-function parseFrontmatter(
+export function parseFrontmatter(
   raw: string,
 ): { frontmatter: MemoryFrontmatter; content: string } | null {
   const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
@@ -954,7 +955,7 @@ function inferEntityTypeFromFilename(pathRel: string): string | undefined {
   return KNOWN_ENTITY_FILENAME_PREFIXES.has(candidate) ? candidate : undefined;
 }
 
-function normalizeFrontmatterForPath(frontmatter: MemoryFrontmatter, pathRel: string, content: string = ""): MemoryFrontmatter {
+export function normalizeFrontmatterForPath(frontmatter: MemoryFrontmatter, pathRel: string, content: string = ""): MemoryFrontmatter {
   const normalizedPath = pathRel.split(path.sep).join("/");
   let normalizedFrontmatter = frontmatter;
 
@@ -2222,7 +2223,7 @@ export function buildEntitySchemaCacheKey(entitySchemas?: PluginConfig["entitySc
  * downstream report requires at read time so the limit semantics and
  * the distribution semantics stay consistent.
  */
-function isValidBufferSurpriseEvent(value: unknown): value is BufferSurpriseEvent {
+export function isValidBufferSurpriseEvent(value: unknown): value is BufferSurpriseEvent {
   if (!value || typeof value !== "object" || Array.isArray(value)) return false;
   const v = value as Record<string, unknown>;
   if (v.event !== "BUFFER_SURPRISE") return false;
@@ -2264,7 +2265,8 @@ export class StorageManager {
   private artifactIndexCache: { memories: MemoryFile[]; loadedAtMs: number; writeVersion: number } | null = null;
   /** Read by storage/entity-store.ts (decomposition), hence not `private`. */
   static readonly KNOWLEDGE_INDEX_CACHE_TTL_MS = 600_000; // 10 minutes (entity mutations invalidate)
-  private static readonly ARTIFACT_INDEX_CACHE_TTL_MS = 60_000; // 1 minute
+  /** Read by storage/memory-read-store.ts (decomposition). */
+  static readonly ARTIFACT_INDEX_CACHE_TTL_MS = 60_000; // 1 minute
   private static readonly artifactWriteVersionByDir = new Map<string, number>();
   private static readonly memoryStatusVersionByDir = new Map<string, number>();
   private static readonly secureStoreEntityCacheKeyIds = new WeakMap<Buffer, number>();
@@ -2293,14 +2295,18 @@ export class StorageManager {
   // on every cold-tier write, making the cache correct across process boundaries
   // (gateway + CLI). After Finding UTsP broadened the scan to the entire cold/
   // subtree, amortizing across back-to-back writes is even more important.
-  private static readonly COLD_SCAN_CACHE_TTL_MS = 30_000; // 30 seconds
-  private static readonly coldMemoriesCache = new Map<string, { memories: MemoryFile[]; loadedAt: number; coldVersion: number }>();
+  /** Read by storage/memory-read-store.ts (decomposition). */
+  static readonly COLD_SCAN_CACHE_TTL_MS = 30_000; // 30 seconds
+  /** Read by storage/memory-read-store.ts (decomposition). */
+  static readonly coldMemoriesCache = new Map<string, { memories: MemoryFile[]; loadedAt: number; coldVersion: number }>();
 
   // Cache for readQuestions() — avoids serially re-reading tens of thousands of
   // question files on every recall.  60-second TTL is intentionally short so that
   // newly written questions surface quickly.
-  private static readonly QUESTIONS_CACHE_TTL_MS = 60_000; // 1 minute
-  private static readonly questionsCache = new Map<
+  /** Read by storage/memory-read-store.ts (decomposition). */
+  static readonly QUESTIONS_CACHE_TTL_MS = 60_000; // 1 minute
+  /** Read by storage/memory-read-store.ts (decomposition). */
+  static readonly questionsCache = new Map<
     string,
     {
       questions: Array<{
@@ -2516,7 +2522,32 @@ export class StorageManager {
     this.loadAliasesSync();
   }
 
+  /**
+   * The LIVE class object of this instance (issue #1809 review of the
+   * storage decomposition): extracted store modules must read/write the
+   * shared static caches (questionsCache, coldMemoriesCache, ...) through
+   * the SAME class binding the host methods use. Referencing the imported
+   * `StorageManager` symbol from a split bundle chunk can yield a second
+   * class copy with its own statics, split-braining the caches
+   * (resolveQuestion clears copy A while readQuestions reads copy B).
+   */
+  get storageManagerClass(): typeof StorageManager {
+    return this.constructor as typeof StorageManager;
+  }
+
   /** The root directory of this storage instance. */
+  /** MemoryReadStore (storage.ts decomposition). Lazy; selfDeps live wiring. */
+  private _memoryReadStore: MemoryReadStore | undefined;
+
+  private get memoryReadStore(): MemoryReadStore {
+    if (!this._memoryReadStore) {
+      this._memoryReadStore = new MemoryReadStore(
+        selfDeps<ConstructorParameters<typeof MemoryReadStore>[0]>(this),
+      );
+    }
+    return this._memoryReadStore;
+  }
+
   /** EntityStore (storage.ts decomposition). Lazy; selfDeps live wiring. */
   private _entityStore: EntityStore | undefined;
 
@@ -2660,52 +2691,12 @@ export class StorageManager {
     }
   }
 
-  /**
-   * List stored transcript days, newest first, optionally scoped to one
-   * source. Non-transcript files in the tree are ignored.
-   */
   async listWearableTranscriptDays(
     sourceId?: string,
   ): Promise<Array<{ source: string; date: string }>> {
-    const days: Array<{ source: string; date: string }> = [];
-    let sources: string[];
-    if (sourceId !== undefined) {
-      sources = [sourceId];
-    } else {
-      try {
-        const entries = await readdir(this.wearablesDir, { withFileTypes: true });
-        sources = entries
-          .filter((entry) => entry.isDirectory())
-          .map((entry) => entry.name);
-      } catch (err) {
-        if ((err as NodeJS.ErrnoException).code === "ENOENT") return [];
-        throw err;
-      }
-    }
-    for (const source of sources) {
-      if (!/^[a-z][a-z0-9-]{0,63}$/.test(source)) continue;
-      let entries: string[];
-      try {
-        entries = await readdir(path.join(this.wearablesDir, source));
-      } catch (err) {
-        if ((err as NodeJS.ErrnoException).code === "ENOENT") continue;
-        throw err;
-      }
-      for (const entry of entries) {
-        if (!entry.endsWith(".md")) continue;
-        const date = entry.slice(0, -3);
-        if (!isValidTranscriptDate(date)) continue;
-        days.push({ source, date });
-      }
-    }
-    days.sort((a, b) => {
-      if (a.date > b.date) return -1;
-      if (a.date < b.date) return 1;
-      if (a.source < b.source) return -1;
-      if (a.source > b.source) return 1;
-      return 0;
-    });
-    return days;
+    return this.memoryReadStore.listWearableTranscriptDays(
+      sourceId,
+    );
   }
 
   /**
@@ -4044,55 +4035,8 @@ export class StorageManager {
   }
 
   private async readAllArtifactsCached(): Promise<MemoryFile[]> {
-    if (
-      this.artifactIndexCache &&
-      Date.now() - this.artifactIndexCache.loadedAtMs <= StorageManager.ARTIFACT_INDEX_CACHE_TTL_MS &&
-      this.artifactIndexCache.writeVersion === this.getArtifactWriteVersion()
-    ) {
-      return this.artifactIndexCache.memories;
-    }
-
-    const scanArtifacts = async (): Promise<MemoryFile[]> => {
-      const artifacts: MemoryFile[] = [];
-      const readDir = async (dir: string) => {
-        try {
-          const entries = await readdir(dir, { withFileTypes: true });
-          for (const entry of entries) {
-            const fullPath = path.join(dir, entry.name);
-            if (entry.isDirectory()) {
-              await readDir(fullPath);
-              continue;
-            }
-            if (!entry.name.endsWith(".md")) continue;
-            const memory = await this.readMemoryByPath(fullPath);
-            if (!memory) continue;
-            artifacts.push(memory);
-          }
-        } catch {
-          // Directory doesn't exist yet
-        }
-      };
-      await readDir(this.artifactsDir);
-      return artifacts;
-    };
-
-    const MAX_REBUILD_RETRIES = 2;
-    let latestArtifacts: MemoryFile[] = [];
-    for (let attempt = 0; attempt <= MAX_REBUILD_RETRIES; attempt += 1) {
-      const versionBefore = this.getArtifactWriteVersion();
-      const artifacts = await scanArtifacts();
-      const versionAfter = this.getArtifactWriteVersion();
-      latestArtifacts = artifacts;
-      if (versionAfter === versionBefore) {
-        this.artifactIndexCache = { memories: artifacts, loadedAtMs: Date.now(), writeVersion: versionAfter };
-        return artifacts;
-      }
-    }
-
-    // Highly concurrent writer churn; keep cache invalid so next read retries a clean rebuild.
-    // Return best-effort latest scan instead of an empty set to avoid dropping recall entirely.
-    this.artifactIndexCache = null;
-    return latestArtifacts;
+    return this.memoryReadStore.readAllArtifactsCached(
+    );
   }
 
   async searchArtifacts(query: string, maxResults: number): Promise<MemoryFile[]> {
@@ -4363,98 +4307,8 @@ export class StorageManager {
   }
 
   private async collectActiveMemoryPaths(): Promise<string[]> {
-    const filePaths: string[] = [];
-
-    // Resolve the memory root once for containment checks below. A category dir
-    // symlinked outside memoryDir (e.g. decisions/ -> an external dir) must NOT
-    // pull out-of-store files into the QMD-unavailable recall fallback (info
-    // leak). Same walker-hardening pattern as document-scanner.ts / cli.ts /
-    // consolidation-provenance-check.ts; reuses the shared containment helper.
-    let memoryRootReal: string;
-    try {
-      memoryRootReal = await realpath(this.baseDir);
-    } catch {
-      return filePaths;
-    }
-
-    const collectPaths = async (dir: string) => {
-      // Directory-level guard, isolated from per-entry handling: skip symlinked
-      // or non-directory category dirs and assert the resolved dir stays inside
-      // the memory root before reading. A failure here means the whole subtree
-      // does not exist or escaped the store — fail closed by skipping it.
-      let entries: Dirent[];
-      try {
-        const dirStat = await lstat(dir);
-        if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) return;
-        assertPathInsideRoot(memoryRootReal, await realpath(dir), dir);
-        entries = await readdir(dir, { withFileTypes: true });
-      } catch {
-        return;
-      }
-
-      const subdirs: string[] = [];
-      for (const entry of entries) {
-        // Never follow symlinked entries out of the store.
-        if (entry.isSymbolicLink()) continue;
-        const fullPath = path.join(dir, entry.name);
-        if (entry.isDirectory()) {
-          subdirs.push(fullPath);
-        } else if (entry.name.endsWith(".md")) {
-          // Isolate per-entry failures in their own try/catch: a containment or
-          // realpath failure on ONE .md entry must not drop sibling files or,
-          // crucially, the deferred subdir recursion below (Cursor Bugbot:
-          // "Poisoned md skips sibling subdirs"). Mirrors the per-file try/catch
-          // in search/document-scanner.ts scanDir and
-          // consolidation-provenance-check.ts walkMarkdownFiles.
-          try {
-            assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
-            filePaths.push(fullPath);
-          } catch {
-            // Skip just this entry (symlink/containment/realpath failure).
-          }
-        }
-      }
-      // Recurse into real subdirectories regardless of any single poisoned entry
-      // above, so valid nested in-store memories are never dropped.
-      for (const subdir of subdirs) {
-        await collectPaths(subdir);
-      }
-    };
-
-    // Scan EVERY supported memory category directory, not just the legacy four
-    // (facts/procedures/reasoning-traces/corrections). Issue #1497: the QMD
-    // filesystem-fallback recall path (orchestrator `recent_scan` ->
-    // readAllMemoriesForNamespaces -> readAllMemories -> here) must read every
-    // recall category dir so on-disk memories in preferences/decisions/moments/
-    // commitments/principles/rules/skills/relationships are not missed when QMD
-    // is disabled, missing, or unhealthy. RECALL_FALLBACK_DIRS is the single
-    // source of truth derived from ALL_CATEGORY_DIRS (shared with
-    // ensureDirectories() and the write routing in utils/category-dir.ts).
-    // These paths resolve identically to the legacy this.factsDir /
-    // this.correctionsDir / this.proceduresDir / this.reasoningTracesDir
-    // getters (all `path.join(this.baseDir, <dir>)`), so the scan stays
-    // namespace-aware: this.baseDir is per-namespace, set by the storage router.
-    // Deliberately EXCLUDED (issue #1497 + PR #1503 review): the non-category
-    // content dirs that ensureDirectories() also creates — entities/, state/,
-    // artifacts/, identity/, config/ — plus the root profile.md, AND the
-    // questions/ queue dir. questions/ holds operational question-QUEUE items
-    // written by writeQuestion() (frontmatter `{ id, created, priority,
-    // resolved }`), read only via readQuestions() and surfaced through the
-    // dedicated, disabled-by-default `injectQuestions` recall-pipeline stage —
-    // never as standard recall memories. The QMD primary recall corpus does not
-    // include them, so the fallback must not either (corpus parity; CLAUDE.md
-    // rule #39). Were questions/ scanned here, parseFrontmatter() would accept
-    // those files (they have a `---` frontmatter block) and leak queue items
-    // into recall. None of these excluded dirs are in RECALL_FALLBACK_DIRS; the
-    // exclusion is asserted by tests in storage-fallback-category-dirs.test.ts.
-    //
-    // collectPaths() already ignores missing dirs (try/catch) and the parser
-    // returns null for non-memory markdown, so unrelated files never crash the
-    // scan.
-    for (const dir of RECALL_FALLBACK_DIRS) {
-      await collectPaths(path.join(this.baseDir, dir));
-    }
-    return filePaths;
+    return this.memoryReadStore.collectActiveMemoryPaths(
+    );
   }
 
   private async readParsedMemoriesFromPaths(
@@ -4598,52 +4452,9 @@ export class StorageManager {
     batchSize?: number;
     updatedAfter?: Date;
   } = {}): Promise<{ memories: MemoryFile[]; filePaths: string[] }> {
-    const allPaths = await this.collectActiveMemoryPaths();
-    const sortedPaths = this.orderWindowPaths(allPaths);
-    const maxMemories =
-      typeof options.maxMemories === "number" && Number.isFinite(options.maxMemories)
-        ? Math.max(1, Math.floor(options.maxMemories))
-        : undefined;
-    const maxCandidatePaths = maxMemories === undefined ? undefined : maxMemories * 2;
-    const updatedAfterMs = options.updatedAfter?.getTime();
-    const normalizedBatchSize = this.normalizeMemoryReadBatchSize(options.batchSize);
-    const memories: MemoryFile[] = [];
-    const selectedPaths: string[] = [];
-
-    for (let i = 0; i < sortedPaths.length; i += normalizedBatchSize) {
-      if (
-        maxMemories !== undefined
-        && (memories.length >= maxMemories || (maxCandidatePaths !== undefined && selectedPaths.length >= maxCandidatePaths))
-      ) {
-        return { memories, filePaths: selectedPaths };
-      }
-      const batchPaths = sortedPaths.slice(i, i + normalizedBatchSize);
-      const candidateBatchPaths = updatedAfterMs === undefined
-        ? batchPaths
-        : await this.filterWindowPathsByUpdatedAfter(batchPaths, updatedAfterMs);
-      const remainingSlots = maxMemories === undefined ? undefined : Math.max(0, maxMemories - memories.length);
-      const remainingInspectionBudget = maxCandidatePaths === undefined ? undefined : Math.max(0, maxCandidatePaths - selectedPaths.length);
-      const { memories: batchMemories, filePaths: parsedCandidatePaths } = remainingSlots === undefined
-        ? {
-            memories: await this.readParsedMemoriesFromPaths(candidateBatchPaths, normalizedBatchSize),
-            filePaths: candidateBatchPaths,
-          }
-        : await this.readWindowBoundedBatch(
-            candidateBatchPaths,
-            remainingSlots,
-            remainingInspectionBudget ?? remainingSlots,
-            normalizedBatchSize,
-          );
-      selectedPaths.push(...parsedCandidatePaths);
-      for (const memory of batchMemories) {
-        memories.push(memory);
-        if (maxMemories !== undefined && memories.length >= maxMemories) {
-          return { memories, filePaths: selectedPaths };
-        }
-      }
-    }
-
-    return { memories, filePaths: selectedPaths };
+    return this.memoryReadStore.readMemoriesWindow(
+      options,
+    );
   }
 
   private async _readAllMemoriesFromDisk(): Promise<MemoryFile[]> {
@@ -4651,85 +4462,9 @@ export class StorageManager {
     return this.readParsedMemoriesFromPaths(filePaths, 50);
   }
 
-  /**
-   * Read all memories from the cold tier by scanning the entire cold/ root
-   * tree.  Previously this only scanned cold/facts/ and cold/corrections/, but
-   * structuredAttributes can appear on any MemoryCategory (preference, decision,
-   * entity, etc.).  buildTierMemoryPath now routes each category to its own
-   * cold/<dir>/ subtree via the shared categoryDirName() chokepoint (issue
-   * #1546), so cold decisions/preferences/... live outside cold/facts/.
-   * Scanning the full coldRoot covers every category dir and guards against
-   * files placed in unexpected subdirectories during manual operations or future
-   * refactors.
-   *
-   * Broadened in PR #402 round-6 (Finding UTsP): scanning only facts/ and
-   * corrections/ was a narrower-than-necessary subset of the cold directory
-   * tree.  Correctness trumps the minor performance difference — cold scans
-   * already happen at most once per supersession write.
-   *
-   * Used by applyTemporalSupersession so that memories already demoted to
-   * cold/ can still be marked superseded when a newer hot fact arrives.
-   *
-   * Cached with a TTL (Finding UOGi, PR #402 round-6): back-to-back
-   * structured-attribute writes in the same burst reuse the cached result
-   * instead of re-scanning the cold tree on every call.  The cache is
-   * invalidated whenever a write calls invalidateAllMemoriesCache() (which
-   * covers any hot→cold demotion that changes cold-tier contents) and
-   * expires after COLD_SCAN_CACHE_TTL_MS as a safety net.
-   */
   async readAllColdMemories(): Promise<MemoryFile[]> {
-    const coldRoot = this.resolveTierRootDir("cold");
-
-    // Read the on-disk cold-version sentinel BEFORE checking the cache so that
-    // writes made by other processes (gateway + CLI) are detected immediately.
-    // Finding UvUy (PR #402 round-11): without this check the cache served
-    // stale data for up to 30s when another process wrote a new cold memory.
-    const currentColdVersion = this.readColdWriteVersion();
-
-    // Return cached result if still valid by both TTL and sentinel version.
-    const cached = StorageManager.coldMemoriesCache.get(coldRoot);
-    if (
-      cached &&
-      Date.now() - cached.loadedAt < StorageManager.COLD_SCAN_CACHE_TTL_MS &&
-      cached.coldVersion === currentColdVersion
-    ) {
-      return cached.memories;
-    }
-
-    const filePaths: string[] = [];
-
-    const collectPaths = async (dir: string) => {
-      try {
-        const entries = await readdir(dir, { withFileTypes: true });
-        const subdirs: string[] = [];
-        for (const entry of entries) {
-          const fullPath = path.join(dir, entry.name);
-          if (entry.isDirectory()) {
-            subdirs.push(fullPath);
-          } else if (entry.name.endsWith(".md")) {
-            filePaths.push(fullPath);
-          }
-        }
-        for (const subdir of subdirs) {
-          await collectPaths(subdir);
-        }
-      } catch {
-        // Directory does not exist yet — cold tier may be empty.
-      }
-    };
-
-    // Scan the entire cold root so that memories in any subdirectory (facts/,
-    // corrections/, artifacts/, or any future category-specific subdirectory)
-    // are included.  This is broader than the previous facts/+corrections/ scan
-    // and ensures that any memory with structuredAttributes is found regardless
-    // of which category it was written with.
-    await collectPaths(coldRoot);
-    const memories = await this.readParsedMemoriesFromPaths(filePaths, 50);
-
-    // Store in cache with the sentinel version captured above so that any
-    // subsequent cold-version bump (by this or another process) invalidates it.
-    StorageManager.coldMemoriesCache.set(coldRoot, { memories, loadedAt: Date.now(), coldVersion: currentColdVersion });
-    return memories;
+    return this.memoryReadStore.readAllColdMemories(
+    );
   }
 
   /**
@@ -4779,67 +4514,10 @@ export class StorageManager {
     return memories;
   }
 
-  /** Read a single memory file by its absolute path. Returns null if unreadable. */
   async readMemoryByPath(filePath: string): Promise<MemoryFile | null> {
-    try {
-      const raw = await readMaybeEncryptedFile(filePath, this._secureStoreKey, this.baseDir);
-      // Note: the outer catch intentionally swallows most errors (ENOENT etc.)
-      // but SecureStoreLockedError must propagate — see re-throw below.
-      const parsed = parseFrontmatter(raw);
-      if (parsed) {
-        return {
-          path: filePath,
-          frontmatter: normalizeFrontmatterForPath(
-            parsed.frontmatter,
-            toMemoryPathRel(this.baseDir, filePath),
-            parsed.content,
-          ),
-          content: parsed.content,
-        };
-      }
-
-      // Entity files use a `# Name` + `**Type:** ...` markdown format rather than
-      // YAML frontmatter. Build a synthetic MemoryFile so entity files returned by
-      // the direct retrieval agent participate in boostSearchResults and last-recall
-      // tracking rather than being silently dropped.
-      const normalizedPath = filePath.split(path.sep).join("/");
-      if (normalizedPath.includes("/entities/") && filePath.endsWith(".md")) {
-        const entity = parseEntityFile(raw, this.entitySchemas);
-        if (!entity.name) return null;
-        const nameWithoutExt = path.basename(filePath, ".md");
-        // Fall back to file mtime rather than new Date() so that entities without
-        // an explicit Updated: timestamp are not treated as freshly created on every
-        // read. Using new Date() would inflate boostSearchResults recency scores for
-        // every entity that lacks a timestamp.
-        // Use epoch as the last-resort fallback so that entities without a
-        // parseable timestamp don't appear as "freshly created" and inflate scores.
-        const fileMtime = entity.updated
-          || await stat(filePath).then((s) => s.mtime.toISOString()).catch(() => new Date(0).toISOString());
-        return {
-          path: filePath,
-          frontmatter: {
-            id: nameWithoutExt,
-            category: "entity",
-            created: fileMtime,
-            updated: fileMtime,
-            source: "entity_extraction",
-            confidence: 0.9,
-            confidenceTier: confidenceTier(0.9),
-            tags: entity.type ? [entity.type] : [],
-          },
-          content: raw,
-        };
-      }
-
-      return null;
-    } catch (err) {
-      // Re-throw store-locked errors — callers need to distinguish "locked"
-      // from "file not found / parse error". Swallowing a locked error here
-      // would silently return null and leave the daemon appearing to work
-      // while returning no memories (subtle data loss).
-      if (err instanceof SecureStoreLockedError) throw err;
-      return null;
-    }
+    return this.memoryReadStore.readMemoryByPath(
+      filePath,
+    );
   }
 
   private resolveTierRootDir(tier: "hot" | "cold"): string {
@@ -5340,130 +5018,20 @@ export class StorageManager {
     return events.length;
   }
 
-  /**
-   * Append a batch of `BUFFER_SURPRISE` telemetry events (issue #563 PR 3).
-   *
-   * Each event records a single buffer flush decision driven by the
-   * surprise gate. The ledger is consumed by
-   * `reportBufferSurpriseDistribution` (Doctor report) and by downstream
-   * benchmark analysis. This method is fire-and-forget by contract:
-   * callers log but do not fail the hot path if the append throws.
-   */
   async appendBufferSurpriseEvents(
     events: BufferSurpriseEvent[],
   ): Promise<number> {
-    if (events.length === 0) return 0;
-    await this.ensureDirectories();
-
-    const nowIso = new Date().toISOString();
-    const payload = events
-      .map((event) => {
-        const normalized: BufferSurpriseEvent = {
-          ...event,
-          event: "BUFFER_SURPRISE",
-          timestamp:
-            event.timestamp && event.timestamp.length > 0
-              ? event.timestamp
-              : nowIso,
-        };
-        return `${JSON.stringify(normalized)}\n`;
-      })
-      .join("");
-
-    await this.appendStorageSecureFile(this.bufferSurpriseLedgerPath, payload);
-    return events.length;
+    return this.memoryReadStore.appendBufferSurpriseEvents(
+      events,
+    );
   }
 
-  /**
-   * Read the buffer-surprise ledger, most recent rows last.
-   *
-   * `limit` bounds the number of **valid rows** returned (not the
-   * number of raw lines parsed). We parse every row, discard malformed
-   * ones, then take the tail — so a partial/truncated trailing line
-   * (the common failure mode after an interrupted append) cannot hide
-   * otherwise-valid recent data above it.
-   *
-   * Non-positive / non-integer / non-finite limits return `[]` rather
-   * than the entire file, matching the other ledger readers in this
-   * class and protecting against `slice(-0.5)` → `slice(-0)` silently
-   * devolving into an unbounded parse.
-   *
-   * # Performance note
-   *
-   * For very large ledgers (issue #563 follow-up), a tail-first reader
-   * would avoid parsing the full file when only a recent window is
-   * needed. We keep the full-scan implementation here because:
-   *
-   *   - the ledger is opt-in (flag off by default), so early deployments
-   *     accumulate rows slowly;
-   *   - telemetry rows are small (~200 bytes), so even 100k rows parse
-   *     in well under a second;
-   *   - the governance archive/cleanup flow can trim the ledger when
-   *     size becomes a concern, reusing the existing maintenance hooks.
-   *
-   * Swap to a chunked tail-reader if production logs show this is a
-   * hot path — leaving that work for a follow-up keeps this PR scoped
-   * to correctness, not optimization.
-   */
   async readBufferSurpriseEvents(
     options: { limit?: number } = {},
   ): Promise<BufferSurpriseEvent[]> {
-    let raw: string;
-    try {
-      raw = await this.readStorageSecureFile(this.bufferSurpriseLedgerPath);
-    } catch (err) {
-      if (err instanceof SecureStoreLockedError) throw err;
-      const code = (err as NodeJS.ErrnoException).code;
-      if (code === "ENOENT") return [];
-      throw err;
-    }
-
-    // Resolve the effective limit up front. Any non-finite / non-positive
-    // value returns no rows — callers who want "everything" should OMIT
-    // the `limit` key (treated as "no bound" below). We intentionally
-    // reject `Infinity` too, because the slice math `events.slice(-Inf)`
-    // is surprising and ambiguous; omit the key instead. Fractional
-    // values <1 floor to 0, which would make `slice(-0)` return the
-    // entire file — guard against that too.
-    let effectiveLimit: number | null = null;
-    if (options.limit !== undefined) {
-      if (
-        typeof options.limit !== "number" ||
-        !Number.isFinite(options.limit) ||
-        options.limit <= 0
-      ) {
-        return [];
-      }
-      const floored = Math.floor(options.limit);
-      if (floored <= 0) return [];
-      effectiveLimit = floored;
-    }
-
-    const lines = raw.split("\n");
-    const events: BufferSurpriseEvent[] = [];
-    for (const line of lines) {
-      const trimmed = line.trim();
-      if (trimmed.length === 0) continue;
-      try {
-        const parsed = JSON.parse(trimmed);
-        if (isValidBufferSurpriseEvent(parsed)) {
-          events.push(parsed);
-        }
-      } catch {
-        // Malformed row — fail open, skip.
-      }
-    }
-
-    events.sort(
-      (left, right) => Date.parse(left.timestamp) - Date.parse(right.timestamp),
+    return this.memoryReadStore.readBufferSurpriseEvents(
+      options,
     );
-
-    if (effectiveLimit === null) return events;
-    // Slice over VALID rows, not raw lines, so malformed tails cannot
-    // mask good data above them. Sort by event timestamp before slicing
-    // so concurrent probe completion order cannot make an older scored
-    // turn look newer than a later scored turn.
-    return events.slice(-effectiveLimit);
   }
 
   async appendBehaviorSignals(events: BehaviorSignalEvent[]): Promise<number> {
@@ -5781,106 +5349,9 @@ export class StorageManager {
   private async readCompressionGuidelineStateFile(
     filePath: string,
   ): Promise<CompressionGuidelineOptimizerState | null> {
-    const isFiniteNonNegativeInteger = (value: unknown): value is number =>
-      typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value >= 0;
-    const isValidActionSummary = (
-      value: unknown,
-    ): value is NonNullable<CompressionGuidelineOptimizerState["actionSummaries"]>[number] => {
-      if (!value || typeof value !== "object") return false;
-      const summary = value as NonNullable<CompressionGuidelineOptimizerState["actionSummaries"]>[number];
-      return (
-        typeof summary.action === "string" &&
-        isFiniteNonNegativeInteger(summary.total) &&
-        summary.outcomes !== null &&
-        typeof summary.outcomes === "object" &&
-        isFiniteNonNegativeInteger(summary.outcomes.applied) &&
-        isFiniteNonNegativeInteger(summary.outcomes.skipped) &&
-        isFiniteNonNegativeInteger(summary.outcomes.failed) &&
-        summary.quality !== null &&
-        typeof summary.quality === "object" &&
-        isFiniteNonNegativeInteger(summary.quality.good) &&
-        isFiniteNonNegativeInteger(summary.quality.poor) &&
-        isFiniteNonNegativeInteger(summary.quality.unknown)
-      );
-    };
-    const isValidRuleUpdate = (
-      value: unknown,
-    ): value is NonNullable<CompressionGuidelineOptimizerState["ruleUpdates"]>[number] => {
-      if (!value || typeof value !== "object") return false;
-      const rule = value as NonNullable<CompressionGuidelineOptimizerState["ruleUpdates"]>[number];
-      return (
-        typeof rule.action === "string" &&
-        typeof rule.delta === "number" &&
-        Number.isFinite(rule.delta) &&
-        (rule.direction === "increase" || rule.direction === "decrease" || rule.direction === "hold") &&
-        (rule.confidence === "low" || rule.confidence === "medium" || rule.confidence === "high") &&
-        Array.isArray(rule.notes) &&
-        rule.notes.every((note) => typeof note === "string")
-      );
-    };
-
-    try {
-      const raw = await this.readStorageSecureFile(filePath);
-      const parsed = JSON.parse(raw) as Partial<CompressionGuidelineOptimizerState>;
-      const sourceWindow = parsed?.sourceWindow as Partial<CompressionGuidelineOptimizerState["sourceWindow"]>;
-      const eventCounts = parsed?.eventCounts as Partial<CompressionGuidelineOptimizerState["eventCounts"]>;
-      const activationState =
-        parsed?.activationState === "draft" || parsed?.activationState === "active"
-          ? parsed.activationState
-          : undefined;
-      const contentHash =
-        typeof parsed?.contentHash === "string" && parsed.contentHash.length > 0
-          ? parsed.contentHash
-          : undefined;
-      const actionSummaries = Array.isArray(parsed?.actionSummaries)
-        ? parsed.actionSummaries.filter(isValidActionSummary)
-        : undefined;
-      const ruleUpdates = Array.isArray(parsed?.ruleUpdates)
-        ? parsed.ruleUpdates.filter(isValidRuleUpdate)
-        : undefined;
-      if (
-        !isFiniteNonNegativeInteger(parsed?.version) ||
-        typeof parsed?.updatedAt !== "string" ||
-        parsed.updatedAt.length === 0 ||
-        !sourceWindow ||
-        typeof sourceWindow.from !== "string" ||
-        sourceWindow.from.length === 0 ||
-        typeof sourceWindow.to !== "string" ||
-        sourceWindow.to.length === 0 ||
-        !eventCounts ||
-        !isFiniteNonNegativeInteger(eventCounts.total) ||
-        !isFiniteNonNegativeInteger(eventCounts.applied) ||
-        !isFiniteNonNegativeInteger(eventCounts.skipped) ||
-        !isFiniteNonNegativeInteger(eventCounts.failed) ||
-        !isFiniteNonNegativeInteger(parsed?.guidelineVersion)
-      ) {
-        return null;
-      }
-
-      return {
-        version: parsed.version,
-        updatedAt: parsed.updatedAt,
-        sourceWindow: {
-          from: sourceWindow.from,
-          to: sourceWindow.to,
-        },
-        eventCounts: {
-          total: eventCounts.total,
-          applied: eventCounts.applied,
-          skipped: eventCounts.skipped,
-          failed: eventCounts.failed,
-        },
-        guidelineVersion: parsed.guidelineVersion,
-        ...(contentHash ? { contentHash } : {}),
-        ...(activationState ? { activationState } : {}),
-        ...(actionSummaries ? { actionSummaries } : {}),
-        ...(ruleUpdates ? { ruleUpdates } : {}),
-      };
-    } catch (err) {
-      if (err instanceof SecureStoreLockedError) throw err;
-      if (!isErrnoCode(err, "ENOENT")) throw err;
-      return null;
-    }
+    return this.memoryReadStore.readCompressionGuidelineStateFile(
+      filePath,
+    );
   }
 
   async writeIdentityAnchor(content: string): Promise<void> {
@@ -6109,40 +5580,9 @@ export class StorageManager {
       filePath: string;
     }>
   > {
-    const cacheKey = this.questionsDir;
-    const cached = StorageManager.questionsCache.get(cacheKey);
-    if (cached && Date.now() - cached.loadedAt < StorageManager.QUESTIONS_CACHE_TTL_MS) {
-      // Check dir mtime for cross-process invalidation — if another process
-      // wrote/resolved a question, the directory mtime will be newer than loadedAt.
-      try {
-        const dirStat = await stat(this.questionsDir);
-        if (dirStat.mtimeMs <= cached.loadedAt) {
-          const all = cached.questions;
-          return opts?.unresolvedOnly ? all.filter((q) => !q.resolved) : all;
-        }
-      } catch {
-        // Dir doesn't exist — fall through to re-read
-      }
-    }
-
-    try {
-      const files = await readdir(this.questionsDir);
-      const questions = [];
-      for (const file of files) {
-        if (!file.endsWith(".md")) continue;
-        const filePath = path.join(this.questionsDir, file);
-        const raw = await readMaybeEncryptedFile(filePath, this._secureStoreKey, this.baseDir);
-        const parsed = this.parseQuestionFile(raw, filePath);
-        if (parsed) {
-          questions.push(parsed);
-        }
-      }
-      const sorted = questions.sort((a, b) => b.priority - a.priority);
-      StorageManager.questionsCache.set(cacheKey, { questions: sorted, loadedAt: Date.now() });
-      return opts?.unresolvedOnly ? sorted.filter((q) => !q.resolved) : sorted;
-    } catch {
-      return [];
-    }
+    return this.memoryReadStore.readQuestions(
+      opts,
+    );
   }
 
   /** Invalidate the questions cache (call after writing a question). */