@remnic/core 10.0.58 → 10.0.60

package/src/corpus-scan.ts

@@ -0,0 +1,215 @@
+/**
+ * Corpus maintenance scans (the pure, read-only filesystem work the daemon
+ * runs periodically).
+ *
+ * Three scans live here, all of them corpus-sized:
+ *   - the recall-corpus path walk (hot + cold tiers);
+ *   - the entity-entry fingerprint the canonical-id migration reads on every
+ *     entity unlock;
+ *   - the watermark's freshness stat over the newest hot partition.
+ *
+ * They are plain functions over plain inputs so the corpus-scan worker can run
+ * them off the main event loop: `src/corpus-scan-worker.entry.ts` is bundled
+ * into a plain-JavaScript worker entry (`src/corpus-scan-worker.entry.mjs`,
+ * generated — see scripts/sync-corpus-scan-worker.mjs) and the host falls back
+ * to these functions inline when a worker cannot start. One implementation,
+ * two execution sites, identical results.
+ */
+
+import type { Dirent, Stats } from "node:fs";
+import { createHash } from "node:crypto";
+import { lstat, readdir, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { isErrnoCode } from "./utils/errno.js";
+import { isMacOsReplicationJunkBasename } from "./utils/macos-replication-junk.js";
+
+/** An errno-carrying filesystem failure. */
+export type ErrnoFailure = NodeJS.ErrnoException;
+
+/**
+ * A backend READ failure worth propagating from a strict (census) scan: an
+ * errno-coded fs error that is NOT `ENOENT`. `ENOENT` (an absent dir) and
+ * code-less rejections are expected and skipped; `EACCES`/`EIO`/… mean the
+ * corpus is unreadable and a divergence census must fail loud rather than
+ * silently publish a partial count (issue #2156 round-5). Non-strict (recall)
+ * callers ignore this and degrade to a best-effort partial scan.
+ */
+export function isPropagatableReadError(error: ErrnoFailure): boolean {
+  return error.code !== undefined && error.code !== "ENOENT";
+}
+
+export interface CorpusScanWalkOptions {
+  /** Memory root the walk is contained in. */
+  baseDir: string;
+  /** Directories to walk (category roots, cold root, archive root). */
+  startDirs: readonly string[];
+  /** Strict mode: backend read failures propagate instead of being skipped. */
+  propagateReadErrors?: boolean;
+  /**
+   * Cancellation check at every directory/entry boundary. The inline reader
+   * passes the corpus-read abort check; the worker passes its cancel flag.
+   */
+  checkCancelled?(): void;
+}
+
+/**
+ * Recursively collect `*.md` paths under `startDirs`. Paths only — no
+ * frontmatter parse. Never throws for a missing root (an empty result), and
+ * throws in strict mode when a backend read failure makes the corpus
+ * unreadable.
+ *
+ * Containment is established once, not per file: the memory root is resolved
+ * with a single `realpath`, every directory the walk enters is `lstat`ed (a
+ * symlinked or non-directory entry is never followed), and the entry TYPE
+ * comes from `readdir(withFileTypes)` — so a symlinked entry, file or
+ * directory, is skipped without a per-entry `realpath`. Reads refuse
+ * symlinked files at open time with `O_NOFOLLOW`, so a path swapped to a
+ * symlink mid-scan is refused rather than followed.
+ */
+export async function collectContainedMarkdownPaths(options: CorpusScanWalkOptions): Promise<string[]> {
+  const { baseDir, startDirs, propagateReadErrors = false, checkCancelled } = options;
+  const filePaths: string[] = [];
+
+  // Resolve the memory root once: a missing root is an empty corpus and an
+  // unreadable root is a backend failure the strict census must surface.
+  try {
+    await realpath(baseDir);
+  } catch (error) {
+    // SAFETY: node:fs/promises rejects with an Error; the errno `code` is
+    // optional on that type, so this adds no unchecked claim.
+    const failure = error as ErrnoFailure;
+    // ENOENT: a not-yet-created root ⇒ empty. Any other errno is a backend
+    // read failure — propagate for strict census callers (never publish an
+    // unreadable corpus as empty); non-strict (recall) callers degrade.
+    if (propagateReadErrors && isPropagatableReadError(failure)) throw error;
+    return filePaths;
+  }
+
+  const collectPaths = async (dir: string): Promise<void> => {
+    checkCancelled?.();
+    // Directory-level guard, isolated from per-entry handling: skip symlinked
+    // or non-directory category dirs. A failure here means the whole subtree
+    // does not exist or escaped the store — fail closed by skipping it.
+    let dirStat: Stats;
+    try {
+      dirStat = await lstat(dir);
+    } catch (error) {
+      // SAFETY: node:fs/promises rejects with an Error; `code` is optional.
+      const failure = error as ErrnoFailure;
+      // ENOENT (absent category dir) is expected; a backend read error
+      // (EACCES/EIO/…) propagates in strict census mode.
+      if (propagateReadErrors && isPropagatableReadError(failure)) throw error;
+      return;
+    }
+    if (dirStat.isSymbolicLink()) return; // never follow symlinks out of the store (both modes)
+    if (!dirStat.isDirectory()) {
+      // A category root that EXISTS but is not a directory (e.g. `facts/`
+      // replaced by a regular file) is a layout/backend failure: a strict
+      // census must not silently publish the remaining categories as healthy.
+      // Non-strict (recall) stays best-effort and skips it (issue #2156 round-10).
+      if (propagateReadErrors) {
+        throw new Error(`corpus scan: expected a directory but found a non-directory at ${dir}`);
+      }
+      return;
+    }
+    let entries: Dirent[];
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch (error) {
+      // SAFETY: node:fs/promises rejects with an Error; `code` is optional.
+      const failure = error as ErrnoFailure;
+      // A backend read error (EACCES …) propagates in strict census mode so a
+      // partial subtree is not silently counted. ENOENT (vanished mid-scan)
+      // and other non-propagatable failures skip the directory.
+      if (propagateReadErrors && isPropagatableReadError(failure)) throw error;
+      return;
+    }
+
+    const subdirs: string[] = [];
+    for (const entry of entries) {
+      // Outside the per-entry handling below, which deliberately swallows
+      // non-propagatable errors — a cancellation must never be swallowed with
+      // them (issue #2307).
+      checkCancelled?.();
+      // Never follow symlinked entries out of the store.
+      if (entry.isSymbolicLink()) continue;
+      if (isMacOsReplicationJunkBasename(entry.name)) continue;
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        subdirs.push(fullPath);
+      } else if (entry.name.endsWith(".md")) {
+        filePaths.push(fullPath);
+      }
+    }
+    // Recurse into real subdirectories regardless of any single poisoned entry
+    // above, so valid nested in-store memories are never dropped.
+    for (const subdir of subdirs) {
+      await collectPaths(subdir);
+    }
+  };
+
+  for (const dir of startDirs) {
+    await collectPaths(dir);
+  }
+  // A signal that fired during the last directory read would otherwise return a
+  // truncated path list as if it were the whole corpus (issue #2307 review).
+  checkCancelled?.();
+  return filePaths;
+}
+
+/**
+ * Fingerprint of the entity entries a migration's completion verdict depends
+ * on: the sorted entry names plus each entry's identity and timestamps. Two
+ * runs over the same entries produce the same string, and an in-place edit of
+ * an entity file changes it.
+ */
+export async function fingerprintEntityEntries(entitiesDir: string): Promise<string> {
+  let names: string[];
+  try {
+    names = (await readdir(entitiesDir))
+      .filter((name) => path.extname(name) === ".md" && !isMacOsReplicationJunkBasename(name))
+      .sort();
+  } catch (error) {
+    if (isErrnoCode(error, "ENOENT")) return "missing";
+    throw error;
+  }
+  const hash = createHash("sha256");
+  for (const name of names) {
+    hash.update(`${name.length}:${name}:`);
+    try {
+      const entry = await lstat(path.join(entitiesDir, name), { bigint: true });
+      hash.update(`${entry.dev}:${entry.ino}:${entry.size}:${entry.mtimeNs}:${entry.ctimeNs}`);
+    } catch (error) {
+      if (!isErrnoCode(error, "ENOENT")) throw error;
+      hash.update("missing");
+    }
+    hash.update("\n");
+  }
+  return hash.digest("hex");
+}
+
+/**
+ * Stat a batch of files and return their mtimes, `null` for files that
+ * vanished (ENOENT). A backend read failure (EACCES/EIO/…) throws with its
+ * errno code intact so the caller can omit an unreadable corpus instead of
+ * publishing a stale freshness value.
+ */
+export async function statMtimes(paths: readonly string[]): Promise<Array<number | null>> {
+  const results: Array<number | null> = new Array(paths.length).fill(null);
+  // Fixed-size batches: an unbounded Promise.all submits one libuv operation
+  // per file at once, and the worker and main thread share that threadpool, so
+  // a large bulk import would delay every other filesystem operation.
+  const STAT_BATCH_SIZE = 256;
+  for (let start = 0; start < paths.length; start += STAT_BATCH_SIZE) {
+    await Promise.all(paths.slice(start, start + STAT_BATCH_SIZE).map(async (filePath, offset) => {
+      try {
+        const entry = await stat(filePath);
+        results[start + offset] = entry.mtimeMs;
+      } catch (error) {
+        if (isErrnoCode(error, "ENOENT")) return;
+        throw error;
+      }
+    }));
+  }
+  return results;
+}

package/src/corpus-watermark.ts

@@ -37,10 +37,11 @@
  */
 
 import { createHash } from "node:crypto";
-import { stat } from "node:fs/promises";
 import path from "node:path";
+import { setImmediate as yieldToEventLoop } from "node:timers/promises";
 import { capabilityAllowsNamespace, type TokenCapabilities } from "./access-token-capabilities.js";
 import { listNamespaces } from "./namespaces/migrate.js";
+import { corpusScanWorker } from "./corpus-scan-worker-host.js";
 import type { PluginConfig } from "./types.js";
 
 export interface CorpusWatermark {
@@ -67,6 +68,9 @@ export const UNPARTITIONED_BUCKET = "unpartitioned";
 /** Storage tier a memory file lives in. */
 export type CorpusTier = "hot" | "cold";
 
+/** Paths parsed between event-loop yields while building a census. */
+const CENSUS_YIELD_EVERY = 4_096;
+
 const DAY_PARTITION_RE = /^\d{4}-\d{2}-\d{2}$/;
 /** Cold-tier root segment (relative to a namespace base dir): `<baseDir>/cold/...`. */
 const COLD_TIER_DIR = "cold";
@@ -155,7 +159,17 @@ export async function computeCorpusWatermark(input: {
   const census = new Map<string, number>();
   let newestPartition: string | null = null;
   const parsedByPath: ParsedPath[] = [];
+  let sinceYield = 0;
   for (const filePath of paths) {
+    // Parsing one path is cheap, but a census covers the whole corpus (100k+
+    // paths): one uninterrupted pass holds the main thread for hundreds of
+    // milliseconds. Yield to the event loop every CENSUS_YIELD_EVERY paths so
+    // observe/recall interleave; the result is order-independent and identical.
+    if (sinceYield >= CENSUS_YIELD_EVERY) {
+      sinceYield = 0;
+      await yieldToEventLoop();
+    }
+    sinceYield += 1;
     const parsed = parsePartition(baseDir, filePath);
     parsedByPath.push(parsed);
     census.set(parsed.bucket, (census.get(parsed.bucket) ?? 0) + 1);
@@ -171,19 +185,21 @@ export async function computeCorpusWatermark(input: {
 
   let newestWriteAt: string | null = null;
   if (newestPartition !== null) {
-    let maxMtimeMs = Number.NEGATIVE_INFINITY;
+    const newestPaths: string[] = [];
     for (let i = 0; i < paths.length; i += 1) {
       if (parsedByPath[i].tier !== "hot" || parsedByPath[i].date !== newestPartition) continue;
-      try {
-        const st = await stat(paths[i]);
-        if (st.mtimeMs > maxMtimeMs) maxMtimeMs = st.mtimeMs;
-      } catch (err) {
-        // A file vanishing between scan and stat (ENOENT) is an expected race —
-        // skip it. A backend read failure (EACCES/EIO/…) means the corpus is
-        // unreadable: propagate so the namespace is omitted, not published with
-        // a stale/null freshness (issue #2156 round-6).
-        if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
-      }
+      newestPaths.push(paths[i]);
+    }
+    // The freshness sweep is one stat per file in the newest hot partition; it
+    // runs in the corpus-scan worker so the periodic census does not spend
+    // main-thread time on a per-file syscall sweep. A file that vanished
+    // (ENOENT) reports null; a backend read failure (EACCES/EIO/…) propagates
+    // with its errno code so the namespace is omitted, not published with a
+    // stale/null freshness (issue #2156 round-6).
+    const mtimes = await corpusScanWorker.request({ kind: "stat-mtimes", paths: newestPaths });
+    let maxMtimeMs = Number.NEGATIVE_INFINITY;
+    for (const mtimeMs of mtimes) {
+      if (mtimeMs !== null && mtimeMs > maxMtimeMs) maxMtimeMs = mtimeMs;
     }
     if (maxMtimeMs > Number.NEGATIVE_INFINITY) {
       newestWriteAt = new Date(maxMtimeMs).toISOString();

package/src/storage/entity-canonical-id-migration.ts

@@ -13,6 +13,7 @@ import {
 import path from "node:path";
 import { lstatSync, readFileSync } from "node:fs";
 import { z } from "zod";
+import { corpusScanWorker } from "../corpus-scan-worker-host.js";
 import { computeSupersessionKey, normalizeSupersessionKey } from "../temporal-supersession.js";
 import type { EntityFile, MemoryFile } from "../types.js";
 import { log } from "../logger.js";
@@ -102,31 +103,6 @@ async function fingerprintPath(filePath: string): Promise<string> {
     : "missing";
 }
 
-async function fingerprintEntityEntries(entitiesDir: string): Promise<string> {
-  let names: string[];
-  try {
-    names = (await readdir(entitiesDir))
-      .filter((name) => path.extname(name) === ".md" && !isMacOsReplicationJunkBasename(name))
-      .sort();
-  } catch (error) {
-    if (isErrnoCode(error, "ENOENT")) return "missing";
-    throw error;
-  }
-  const hash = createHash("sha256");
-  for (const name of names) {
-    hash.update(`${name.length}:${name}:`);
-    try {
-      const entry = await lstat(path.join(entitiesDir, name), { bigint: true });
-      hash.update(`${entry.dev}:${entry.ino}:${entry.size}:${entry.mtimeNs}:${entry.ctimeNs}`);
-    } catch (error) {
-      if (!isErrnoCode(error, "ENOENT")) throw error;
-      hash.update("missing");
-    }
-    hash.update("\n");
-  }
-  return hash.digest("hex");
-}
-
 /**
  * Completion fingerprint for migration inputs only. Entity entry identities
  * catch same-path edits that do not change the directory inode; the cooperative
@@ -146,7 +122,9 @@ export async function getFingerprint(
     consumingFingerprint,
   ] = await Promise.all([
     fingerprintPath(entitiesDir),
-    fingerprintEntityEntries(entitiesDir),
+    // The entry enumeration is one lstat per entity file — a corpus-sized
+    // syscall storm on the main thread. It runs in the corpus-scan worker.
+    corpusScanWorker.request({ kind: "entity-entries-fingerprint", entitiesDir }),
     fingerprintPath(path.join(baseDir, "config", "aliases.json")),
     fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_MARKER)),
     fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER)),

package/src/storage/memory-read-store.ts

@@ -10,7 +10,7 @@
  */
 
 import type { Dirent, Stats } from "node:fs";
-import { lstat, readdir, realpath, stat } from "node:fs/promises";
+import { lstat, readdir, stat } from "node:fs/promises";
 import path from "node:path";
 import { toMemoryPathRel } from "../memory-lifecycle-ledger-utils.js";
 import { SecureStoreLockedError, readMaybeEncryptedFile } from "../secure-store/secure-fs.js";
@@ -19,7 +19,6 @@ import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
 import { isErrnoCode } from "../utils/errno.js";
 import { isPopulatedUnnamedEntityRecord } from "../utils/malformed-entity-record.js";
 import { isMacOsReplicationJunkBasename } from "../utils/macos-replication-junk.js";
-import { assertPathInsideRoot } from "../utils/path-containment.js";
 import { isAbortError } from "../abort-error.js";
 import { checkCorpusReadAbort, type CorpusReadOptions } from "../corpus-read-cancellation.js";
 import { isValidTranscriptDate } from "../wearables/day-store.js";
@@ -31,6 +30,7 @@ import {
   StorageManager,
 } from "../storage.js";
 import { rememberRawFrontmatter } from "./memory-frontmatter-metadata.js";
+import { corpusScanWorker } from "../corpus-scan-worker-host.js";
 
 export interface MemoryReadStoreDeps {
   /** Live class object of the host instance — shared static caches (see storage.ts storageManagerClass). */
@@ -86,20 +86,6 @@ export interface MemoryReadStoreDeps {
   readonly wearablesDir: string;
 }
 
-/**
- * A backend READ failure worth propagating from a strict (census) scan: an
- * errno-coded fs error that is NOT `ENOENT`. `ENOENT` (an absent dir) and
- * code-less rejections (symlink-escape containment failures from
- * assertPathInsideRoot) are expected and skipped; `EACCES`/`EIO`/… mean the
- * corpus is unreadable and a divergence census must fail loud rather than
- * silently publish a partial count (issue #2156 round-5). Non-strict (recall)
- * callers ignore this and degrade to a best-effort partial scan.
- */
-function isPropagatableReadError(err: unknown): boolean {
-  const code = (err as NodeJS.ErrnoException).code;
-  return typeof code === "string" && code !== "ENOENT";
-}
-
 /**
  * Markdown under this store's `entities/` directory. The first relative
  * segment must be `entities`; a memory root whose absolute path contains
@@ -233,9 +219,13 @@ export class MemoryReadStore {
   /**
    * Recursively collect `*.md` paths under `startDirs`, hardened against symlink
    * escape: a category dir symlinked outside memoryDir (e.g. decisions/ -> an
-   * external dir) must NOT pull out-of-store files into a scan (info leak). Same
-   * walker-hardening pattern as document-scanner.ts / cli.ts /
-   * consolidation-provenance-check.ts; reuses the shared containment helper.
+   * external dir) must NOT pull out-of-store files into a scan (info leak).
+   *
+   * The walk runs in the corpus-scan worker: it is one `readdir` + `lstat` per
+   * directory over the whole corpus, and on the main event loop that starves
+   * observe, recall and the extraction queue behind it. The implementation is
+   * shared (src/corpus-scan.ts) and the host runs it inline when no worker
+   * can start, so the collected path set is the same either way.
    * Paths only — no frontmatter parse.
    */
   private async collectContainedMarkdownPaths(
@@ -243,98 +233,16 @@ export class MemoryReadStore {
     propagateReadErrors = false,
     options?: CorpusReadOptions,
   ): Promise<string[]> {
-    const filePaths: string[] = [];
-
-    // Resolve the memory root once for the containment checks below.
-    let memoryRootReal: string;
-    try {
-      memoryRootReal = await realpath(this.deps.baseDir);
-    } catch (err) {
-      // ENOENT: a not-yet-created root ⇒ empty. Any other errno is a backend
-      // read failure — propagate for strict census callers (never publish an
-      // unreadable corpus as empty); non-strict (recall) callers degrade.
-      if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-      return filePaths;
-    }
-
-    const collectPaths = async (dir: string): Promise<void> => {
-      checkCorpusReadAbort(options);
-      // Directory-level guard, isolated from per-entry handling: skip symlinked
-      // or non-directory category dirs and assert the resolved dir stays inside
-      // the memory root before reading. A failure here means the whole subtree
-      // does not exist or escaped the store — fail closed by skipping it.
-      let dirStat: Stats;
-      try {
-        dirStat = await lstat(dir);
-      } catch (err) {
-        // ENOENT (absent category dir) is expected; a backend read error
-        // (EACCES/EIO/…) propagates in strict census mode.
-        if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-        return;
-      }
-      if (dirStat.isSymbolicLink()) return; // never follow symlinks out of the store (both modes)
-      if (!dirStat.isDirectory()) {
-        // A category root that EXISTS but is not a directory (e.g. `facts/`
-        // replaced by a regular file) is a layout/backend failure: a strict
-        // census must not silently publish the remaining categories as healthy.
-        // Non-strict (recall) stays best-effort and skips it (issue #2156 round-10).
-        if (propagateReadErrors) {
-          throw new Error(`corpus scan: expected a directory but found a non-directory at ${dir}`);
-        }
-        return;
-      }
-      let entries: Dirent[];
-      try {
-        assertPathInsideRoot(memoryRootReal, await realpath(dir), dir);
-        entries = await readdir(dir, { withFileTypes: true });
-      } catch (err) {
-        // A containment failure carries no errno (skip); a backend read error
-        // (EACCES …) propagates in strict census mode so a partial subtree is
-        // not silently counted.
-        if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-        return;
-      }
-
-      const subdirs: string[] = [];
-      for (const entry of entries) {
-        // Outside the per-entry try/catch below, which deliberately swallows
-        // non-propagatable errors — a cancellation must never be swallowed with
-        // them (issue #2307).
-        checkCorpusReadAbort(options);
-        // Never follow symlinked entries out of the store.
-        if (entry.isSymbolicLink()) continue;
-        if (isMacOsReplicationJunkBasename(entry.name)) continue;
-        const fullPath = path.join(dir, entry.name);
-        if (entry.isDirectory()) {
-          subdirs.push(fullPath);
-        } else if (entry.name.endsWith(".md")) {
-          // Isolate per-entry failures in their own try/catch: a containment or
-          // realpath failure on ONE .md entry must not drop sibling files or,
-          // crucially, the deferred subdir recursion below (Cursor Bugbot:
-          // "Poisoned md skips sibling subdirs"). Mirrors the per-file try/catch
-          // in search/document-scanner.ts scanDir and
-          // consolidation-provenance-check.ts walkMarkdownFiles.
-          try {
-            assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
-            filePaths.push(fullPath);
-          } catch (err) {
-            // ENOENT (vanished mid-scan) or a code-less containment/symlink
-            // rejection ⇒ skip just this file. A backend read error propagates
-            // in strict census mode.
-            if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-          }
-        }
-      }
-      // Recurse into real subdirectories regardless of any single poisoned entry
-      // above, so valid nested in-store memories are never dropped.
-      for (const subdir of subdirs) {
-        await collectPaths(subdir);
-      }
-    };
-
-    for (const dir of startDirs) {
-      await collectPaths(dir);
-    }
+    checkCorpusReadAbort(options);
+    const filePaths = await corpusScanWorker.request(
+      {
+        kind: "walk",
+        baseDir: this.deps.baseDir,
+        startDirs: [...startDirs],
+        propagateReadErrors,
+      },
+      options?.abortSignal,
+    );
     // A signal that fired during the last directory read would otherwise return a
     // truncated path list as if it were the whole corpus (issue #2307 review).
     checkCorpusReadAbort(options);