@remnic/core 10.0.59 → 10.0.61

package/src/storage/entity-canonical-id-migration.ts

@@ -13,6 +13,7 @@ import {
 import path from "node:path";
 import { lstatSync, readFileSync } from "node:fs";
 import { z } from "zod";
+import { corpusScanWorker } from "../corpus-scan-worker-host.js";
 import { computeSupersessionKey, normalizeSupersessionKey } from "../temporal-supersession.js";
 import type { EntityFile, MemoryFile } from "../types.js";
 import { log } from "../logger.js";
@@ -102,31 +103,6 @@ async function fingerprintPath(filePath: string): Promise<string> {
     : "missing";
 }
 
-async function fingerprintEntityEntries(entitiesDir: string): Promise<string> {
-  let names: string[];
-  try {
-    names = (await readdir(entitiesDir))
-      .filter((name) => path.extname(name) === ".md" && !isMacOsReplicationJunkBasename(name))
-      .sort();
-  } catch (error) {
-    if (isErrnoCode(error, "ENOENT")) return "missing";
-    throw error;
-  }
-  const hash = createHash("sha256");
-  for (const name of names) {
-    hash.update(`${name.length}:${name}:`);
-    try {
-      const entry = await lstat(path.join(entitiesDir, name), { bigint: true });
-      hash.update(`${entry.dev}:${entry.ino}:${entry.size}:${entry.mtimeNs}:${entry.ctimeNs}`);
-    } catch (error) {
-      if (!isErrnoCode(error, "ENOENT")) throw error;
-      hash.update("missing");
-    }
-    hash.update("\n");
-  }
-  return hash.digest("hex");
-}
-
 /**
  * Completion fingerprint for migration inputs only. Entity entry identities
  * catch same-path edits that do not change the directory inode; the cooperative
@@ -146,7 +122,9 @@ export async function getFingerprint(
     consumingFingerprint,
   ] = await Promise.all([
     fingerprintPath(entitiesDir),
-    fingerprintEntityEntries(entitiesDir),
+    // The entry enumeration is one lstat per entity file — a corpus-sized
+    // syscall storm on the main thread. It runs in the corpus-scan worker.
+    corpusScanWorker.request({ kind: "entity-entries-fingerprint", entitiesDir }),
     fingerprintPath(path.join(baseDir, "config", "aliases.json")),
     fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_MARKER)),
     fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER)),

package/src/storage/memory-read-store.ts

@@ -10,7 +10,7 @@
  */
 
 import type { Dirent, Stats } from "node:fs";
-import { lstat, readdir, realpath, stat } from "node:fs/promises";
+import { lstat, readdir, stat } from "node:fs/promises";
 import path from "node:path";
 import { toMemoryPathRel } from "../memory-lifecycle-ledger-utils.js";
 import { SecureStoreLockedError, readMaybeEncryptedFile } from "../secure-store/secure-fs.js";
@@ -30,6 +30,7 @@ import {
   StorageManager,
 } from "../storage.js";
 import { rememberRawFrontmatter } from "./memory-frontmatter-metadata.js";
+import { corpusScanWorker } from "../corpus-scan-worker-host.js";
 
 export interface MemoryReadStoreDeps {
   /** Live class object of the host instance — shared static caches (see storage.ts storageManagerClass). */
@@ -85,19 +86,6 @@ export interface MemoryReadStoreDeps {
   readonly wearablesDir: string;
 }
 
-/**
- * A backend READ failure worth propagating from a strict (census) scan: an
- * errno-coded fs error that is NOT `ENOENT`. `ENOENT` (an absent dir) and
- * code-less rejections are expected and skipped; `EACCES`/`EIO`/… mean the
- * corpus is unreadable and a divergence census must fail loud rather than
- * silently publish a partial count (issue #2156 round-5). Non-strict (recall)
- * callers ignore this and degrade to a best-effort partial scan.
- */
-function isPropagatableReadError(err: unknown): boolean {
-  const code = (err as NodeJS.ErrnoException).code;
-  return typeof code === "string" && code !== "ENOENT";
-}
-
 /**
  * Markdown under this store's `entities/` directory. The first relative
  * segment must be `entities`; a memory root whose absolute path contains
@@ -233,15 +221,11 @@ export class MemoryReadStore {
    * escape: a category dir symlinked outside memoryDir (e.g. decisions/ -> an
    * external dir) must NOT pull out-of-store files into a scan (info leak).
    *
-   * Containment is established once, not per file: the memory root is resolved
-   * with a single `realpath`, every directory the walk enters is `lstat`ed (a
-   * symlinked or non-directory entry is never followed), and the entry TYPE
-   * comes from `readdir(withFileTypes)` — so a symlinked entry, file or
-   * directory, is skipped without a per-entry `realpath`. The old per-entry
-   * `realpath` cost one syscall per markdown file (hundreds of thousands per
-   * corpus scan) to re-derive a fact the directory-level checks already
-   * establish; reads refuse symlinked files at open time with `O_NOFOLLOW`, so
-   * a path swapped to a symlink mid-scan is refused rather than followed.
+   * The walk runs in the corpus-scan worker: it is one `readdir` + `lstat` per
+   * directory over the whole corpus, and on the main event loop that starves
+   * observe, recall and the extraction queue behind it. The implementation is
+   * shared (src/corpus-scan.ts) and the host runs it inline when no worker
+   * can start, so the collected path set is the same either way.
    * Paths only — no frontmatter parse.
    */
   private async collectContainedMarkdownPaths(
@@ -249,82 +233,16 @@ export class MemoryReadStore {
     propagateReadErrors = false,
     options?: CorpusReadOptions,
   ): Promise<string[]> {
-        const filePaths: string[] = [];
-
-    // Resolve the memory root once: a missing root is an empty corpus and an
-    // unreadable root is a backend failure the strict census must surface.
-    try {
-      await realpath(this.deps.baseDir);
-    } catch (err) {
-      // ENOENT: a not-yet-created root ⇒ empty. Any other errno is a backend
-      // read failure — propagate for strict census callers (never publish an
-      // unreadable corpus as empty); non-strict (recall) callers degrade.
-      if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-      return filePaths;
-    }
-
-    const collectPaths = async (dir: string): Promise<void> => {
-      checkCorpusReadAbort(options);
-      // Directory-level guard, isolated from per-entry handling: skip symlinked
-      // or non-directory category dirs. A failure here means the whole subtree
-      // does not exist or escaped the store — fail closed by skipping it.
-      let dirStat: Stats;
-      try {
-        dirStat = await lstat(dir);
-      } catch (err) {
-        // ENOENT (absent category dir) is expected; a backend read error
-        // (EACCES/EIO/…) propagates in strict census mode.
-        if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-        return;
-      }
-      if (dirStat.isSymbolicLink()) return; // never follow symlinks out of the store (both modes)
-      if (!dirStat.isDirectory()) {
-        // A category root that EXISTS but is not a directory (e.g. `facts/`
-        // replaced by a regular file) is a layout/backend failure: a strict
-        // census must not silently publish the remaining categories as healthy.
-        // Non-strict (recall) stays best-effort and skips it (issue #2156 round-10).
-        if (propagateReadErrors) {
-          throw new Error(`corpus scan: expected a directory but found a non-directory at ${dir}`);
-        }
-        return;
-      }
-      let entries: Dirent[];
-      try {
-        entries = await readdir(dir, { withFileTypes: true });
-      } catch (err) {
-        // A backend read error (EACCES …) propagates in strict census mode so a
-        // partial subtree is not silently counted. ENOENT (vanished mid-scan)
-        // and other non-propagatable failures skip the directory.
-        if (propagateReadErrors && isPropagatableReadError(err)) throw err;
-        return;
-      }
-
-      const subdirs: string[] = [];
-      for (const entry of entries) {
-        // Outside the per-entry handling below, which deliberately swallows
-        // non-propagatable errors — a cancellation must never be swallowed with
-        // them (issue #2307).
-        checkCorpusReadAbort(options);
-        // Never follow symlinked entries out of the store.
-        if (entry.isSymbolicLink()) continue;
-        if (isMacOsReplicationJunkBasename(entry.name)) continue;
-        const fullPath = path.join(dir, entry.name);
-        if (entry.isDirectory()) {
-          subdirs.push(fullPath);
-        } else if (entry.name.endsWith(".md")) {
-          filePaths.push(fullPath);
-        }
-      }
-      // Recurse into real subdirectories regardless of any single poisoned entry
-      // above, so valid nested in-store memories are never dropped.
-      for (const subdir of subdirs) {
-        await collectPaths(subdir);
-      }
-    };
-
-    for (const dir of startDirs) {
-      await collectPaths(dir);
-    }
+    checkCorpusReadAbort(options);
+    const filePaths = await corpusScanWorker.request(
+      {
+        kind: "walk",
+        baseDir: this.deps.baseDir,
+        startDirs: [...startDirs],
+        propagateReadErrors,
+      },
+      options?.abortSignal,
+    );
     // A signal that fired during the last directory read would otherwise return a
     // truncated path list as if it were the whole corpus (issue #2307 review).
     checkCorpusReadAbort(options);

package/src/storage.ts

@@ -144,6 +144,7 @@ import {
 } from "./lifecycle/tombstones.js";
 import { supersessionKeysForFact } from "./temporal-supersession.js";
 import { BufferSessionFiles } from "./storage/buffer-session-files.js";
+import { BufferDrainClaims, type BufferSessionSaveOutcome } from "./storage/buffer-drain-claims.js";
 import { runSupersessionSideEffects, type SupersessionAuditOptions } from "./storage/supersession-side-effects.js";
 import { saveDroppedExtractionChunk as saveDroppedChunk } from "./storage/dropped-extraction-chunk.js";
 import type {
@@ -5144,18 +5145,10 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
             );
           } catch (err) {
             if (priorDest === null) {
-              await recordArchiveDelete(this.baseDir, destPath, () =>
-                unlink(destPath).then(
-                  () => true,
-                  () => false
-                )
-              );
+              await recordArchiveDelete(this.baseDir, destPath, () => unlink(destPath).then(() => true, () => false));
             } else {
               await recordArchiveDelete(this.baseDir, destPath, () =>
-                this.writeStorageSecureFile(destPath, priorDest).then(
-                  () => true,
-                  () => false
-                )
+                this.writeStorageSecureFile(destPath, priorDest).then(() => true, () => false)
               );
             }
             throw err;
@@ -5524,8 +5517,12 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   }
   async loadBuffer(): Promise<BufferState> { return this.bufferSessionFiles().load(); }
   async saveBuffer(state: BufferState): Promise<void> { await this.bufferSessionFiles().reconcile(state); }
-  async saveBufferSession(key: string, entry: BufferEntryState): Promise<void> { await this.bufferSessionFiles().save(key, entry); }
+  async saveBufferSession(key: string, entry: BufferEntryState): Promise<BufferSessionSaveOutcome> { return await this.bufferSessionFiles().save(key, entry); }
   async deleteBufferSession(key: string): Promise<void> { await this.bufferSessionFiles().remove(key); }
+  /** Sync gate check for the extraction queue: is this session claimed by a buffer drain? */
+  isBufferSessionClaimed(key: string): boolean { return this.bufferSessionFiles().isClaimed(key); }
+  /** The drain-claim store for this memory root (CLI drain + worker). */
+  bufferDrainClaims(): BufferDrainClaims { return this.bufferSessionFiles().claims; }
   saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly import("./types.js").BufferTurn[]): Promise<string> { // storage/dropped-extraction-chunk.ts
     return saveDroppedChunk(this.stateDir, (file, content) => this.writeStorageSecureFile(file, content), bufferKey, reason, turns);
   }

package/src/testing/subjects/extraction-lifecycle.test.ts

@@ -17,6 +17,7 @@ import assert from "node:assert/strict";
 import path from "node:path";
 
 import { Orchestrator } from "../../orchestrator.js";
+import { BufferDrainRunner } from "../../orchestration/buffer-drain-runner.js";
 import { resolveNamespaceStorageRoot } from "../../namespaces/storage.js";
 import type { BufferTurn, PluginConfig } from "../../types.js";
 import {
@@ -135,6 +136,37 @@ async function assertHarmonicEpisode(memoryDir: string, sessionKey?: string): Pr
   if (sessionKey) assert.equal(status.latestNode?.sessionKey, sessionKey);
 }
 
+
+/**
+ * The out-of-process drain path: a buffered session is claimed into a drain
+ * claim, the drain extracts it through the same pipeline, and the claim ends
+ * empty. The live buffer keeps its in-memory copy; the drain tombstone makes
+ * the owner subtract the drained turn instead of writing it back.
+ */
+async function exerciseBufferDrain(state: ExtractionLifecycleState): Promise<void> {
+  // Its own memory root and orchestrator: the drain must not perturb the
+  // row's own extraction calls or fact counts.
+  const memoryDir = await mkTempMemoryDir("extraction-drain-row");
+  const cfg = makeHarmonicLifecycleConfig(memoryDir, { extraction: { spanMode: "on" } });
+  const orchestrator = new Orchestrator(cfg);
+  state.orchestrators.push(orchestrator);
+  stubExtraction(orchestrator, (turns) => spanBearingResult(turns));
+  const sessionKey = "drain-row-session";
+  const content = "The drain row retains the canary gate detail for the teal dashboard.";
+  await orchestrator.buffer.addTurn(sessionKey, {
+    role: "user",
+    content,
+    timestamp: new Date().toISOString(),
+    sessionKey,
+  });
+  await orchestrator.buffer.save();
+  const runner = new BufferDrainRunner(orchestrator);
+  assert.equal(await runner.claim(sessionKey, "subject-drain"), true, "the drain claims the buffered session");
+  const outcome = await runner.drainSession(sessionKey, { ownerId: "subject-drain" });
+  assert.equal(outcome.status, "drained", "the drain empties the claimed session");
+  assert.equal(outcome.turnsRemaining, 0, "no turn is left claimed");
+}
+
 const subject: LifecycleSubject<ExtractionLifecycleState> = {
   async setup(row: MatrixRow): Promise<ExtractionLifecycleState> {
     const memoryDir = await mkTempMemoryDir(`extraction-${row.id}`);
@@ -254,6 +286,7 @@ const subject: LifecycleSubject<ExtractionLifecycleState> = {
         return;
       }
       case "before-reset": {
+        await exerciseBufferDrain(state);
         // Buffer a low-signal turn, then fire a before_reset flush whose abort
         // trips mid-extraction. The abort wiring (the signal threaded from
         // flushSession into runExtraction, guarding `before_persist` and
@@ -419,6 +452,7 @@ const subject: LifecycleSubject<ExtractionLifecycleState> = {
         return;
       }
       case "before-reset": {
+        await exerciseBufferDrain(state);
         assert.equal(
           state.abortFlushRejected,
           true,