@remnic/core 10.0.49 → 10.0.51

package/src/storage/entity-migration-completion-cache.ts

@@ -0,0 +1,153 @@
+/**
+ * Amortized entity canonical-id migration completion verdict for entity reads.
+ *
+ * Key covers every getFingerprint input that is cheap to lstat: entity mutation
+ * version, entities/ dir, config/aliases.json, the journal, and reconcile
+ * markers. Full fingerprint (including entity entry enumeration) runs only on
+ * key miss.
+ *
+ * ponytail: in-place entity *content* edits still skip revalidation.
+ */
+
+import { lstat, realpath } from "node:fs/promises";
+import path from "node:path";
+import { SecureStoreLockedError } from "../secure-store/secure-fs.js";
+import { isErrnoCode } from "../utils/errno.js";
+import { pathIsInside } from "../utils/path-containment.js";
+import { getFingerprint } from "./entity-canonical-id-migration.js";
+import {
+  ENTITY_CANONICAL_ID_MIGRATION_FILE,
+  ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER,
+  ENTITY_CANONICAL_ID_RECONCILE_MARKER,
+  entityCanonicalIdMigrationCompleted,
+  resolveHistoricalEntityCanonicalId,
+} from "./entity-canonical-id-references.js";
+
+export type EntityMigrationCompletionCacheDeps = {
+  readonly baseDir: string;
+  readonly entitiesDir: string;
+  getEntityMutationVersion(): number;
+};
+
+/**
+ * True when `entitiesDir` is a real directory inside `baseDir`.
+ * lstat first so a symlinked root is never realpath'd into trust.
+ */
+export async function isTrustedEntitiesRoot(
+  baseDir: string,
+  entitiesDir: string,
+): Promise<boolean> {
+  let rootStat;
+  try {
+    rootStat = await lstat(entitiesDir);
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT")) return false;
+    throw err;
+  }
+  if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) return false;
+  let memoryRoot: string;
+  let entitiesReal: string;
+  try {
+    memoryRoot = await realpath(baseDir);
+    entitiesReal = await realpath(entitiesDir);
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT")) return false;
+    throw err;
+  }
+  return pathIsInside(memoryRoot, entitiesReal);
+}
+
+export class EntityMigrationCompletionCache {
+  private cache: { version: number; inputSig: string; completed: boolean } | undefined;
+
+  /** lstat identity for one fingerprint input path; nofollow; missing → stable token. */
+  private async inputPathSig(filePath: string): Promise<string> {
+    try {
+      const s = await lstat(filePath);
+      return `${s.dev}:${s.ino}:${s.mtimeMs}:${s.ctimeMs}:${s.size}:${s.isSymbolicLink() ? "L" : s.isDirectory() ? "D" : "F"}`;
+    } catch {
+      return "missing";
+    }
+  }
+
+  /**
+   * Cheap coherence key. entities/ absence is uncacheable (random suffix) so
+   * we never pin a false complete.
+   */
+  private async inputSig(deps: EntityMigrationCompletionCacheDeps): Promise<string> {
+    const stateDir = path.join(deps.baseDir, "state");
+    const entitiesSig = await this.inputPathSig(deps.entitiesDir);
+    if (entitiesSig === "missing") return `entities-missing:${Math.random()}`;
+    const [aliasesSig, journalSig, reconcileSig, consumingSig] = await Promise.all([
+      this.inputPathSig(path.join(deps.baseDir, "config", "aliases.json")),
+      this.inputPathSig(path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE)),
+      this.inputPathSig(path.join(stateDir, ENTITY_CANONICAL_ID_RECONCILE_MARKER)),
+      this.inputPathSig(path.join(stateDir, ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER)),
+    ]);
+    return [entitiesSig, aliasesSig, journalSig, reconcileSig, consumingSig].join("|");
+  }
+
+  /**
+   * True when journal completion attests to live fingerprint inputs.
+   * Untrusted entities roots (symlink / outside base) never reach getFingerprint.
+   */
+  async isCompleted(deps: EntityMigrationCompletionCacheDeps): Promise<boolean> {
+    const version = deps.getEntityMutationVersion();
+    const inputSig = await this.inputSig(deps);
+    const cached = this.cache;
+    if (cached && cached.version === version && cached.inputSig === inputSig) return cached.completed;
+    if (!(await isTrustedEntitiesRoot(deps.baseDir, deps.entitiesDir))) {
+      this.cache = { version, inputSig, completed: false };
+      return false;
+    }
+    const completed = await entityCanonicalIdMigrationCompleted(path.join(deps.baseDir, "state"), () =>
+      getFingerprint(deps.baseDir, deps.entitiesDir, () => String(version)));
+    this.cache = { version, inputSig, completed };
+    return completed;
+  }
+}
+
+export type ReadEntityRenameDeps = {
+  readonly entitiesDir: string;
+  currentHistoricalIds(): Readonly<Record<string, string>>;
+  readStorageSecureFile(filePath: string): Promise<string>;
+};
+
+/** Resolve a physical entity path through historical id rename candidates. */
+export async function readEntityFileResolvingRename(
+  deps: ReadEntityRenameDeps,
+  firstPath: string,
+  entityId: string,
+): Promise<{ id: string; content: string } | null> {
+  try {
+    return { id: entityId, content: await deps.readStorageSecureFile(firstPath) };
+  } catch (err) {
+    if (err instanceof SecureStoreLockedError) throw err;
+    // ENOENT: renamed away (retry below). ELOOP: planted symlink — skip.
+    if (isErrnoCode(err, "ELOOP")) return null;
+    if (!isErrnoCode(err, "ENOENT")) throw err;
+  }
+  const historicalIds = deps.currentHistoricalIds();
+  const canonical = resolveHistoricalEntityCanonicalId(entityId, historicalIds);
+  const candidates = new Set([canonical]);
+  for (const legacy of Object.keys(historicalIds)) {
+    if (resolveHistoricalEntityCanonicalId(legacy, historicalIds) === canonical) candidates.add(legacy);
+  }
+  candidates.delete(entityId);
+  for (const candidate of candidates) {
+    try {
+      return {
+        id: candidate,
+        content: await deps.readStorageSecureFile(
+          path.join(deps.entitiesDir, `${candidate}.md`),
+        ),
+      };
+    } catch (err) {
+      if (err instanceof SecureStoreLockedError) throw err;
+      // ENOENT/ELOOP: gone or a planted symlink — try the next candidate.
+      if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "ELOOP")) continue;
+      throw err;
+    }
+  }
+  return null;
+}

package/src/storage/entity-store.ts

@@ -10,11 +10,17 @@
  *   - fragmented-entity merge (consolidation)
  */
 
-import { lstat, readdir, realpath, unlink } from "node:fs/promises";
+import { lstat, readdir, unlink } from "node:fs/promises";
 import path from "node:path";
 import { checkCorpusReadAbort, type CorpusReadOptions } from "../corpus-read-cancellation.js";
 import { normalizeEntityStructuredSection, sortStructuredSectionsBySchema } from "../entity-schema.js";
 import { withEntityCanonicalMutationLock } from "./entity-canonical-id-lock.js";
+import { assertSafeEntityId } from "./entity-id-safety.js";
+import {
+  EntityMigrationCompletionCache,
+  isTrustedEntitiesRoot,
+  readEntityFileResolvingRename,
+} from "./entity-migration-completion-cache.js";
 import { isMalformedEntityRecord } from "../utils/malformed-entity-record.js";
 import { log } from "../logger.js";
 import {
@@ -33,7 +39,6 @@ import type {
   ScoredEntity,
 } from "../types.js";
 import { isErrnoCode } from "../utils/errno.js";
-import { pathIsInside } from "../utils/path-containment.js";
 import {
   reconcileIfJournalMovedSync,
   requestEntityCanonicalIdReconcileSync,
@@ -89,6 +94,7 @@ export interface EntityStoreDeps {
 export class EntityStore {
   /** Filename → file identity of the last indexable check. Unchanged files are not reread. */
   private readonly entityIndexableCache = new Map<string, { identity: string; ok: boolean }>();
+  private readonly migrationCompletion = new EntityMigrationCompletionCache();
 
   constructor(
     private readonly deps: EntityStoreDeps,
@@ -111,6 +117,7 @@ export class EntityStore {
     method: string,
     historicalIds: Readonly<Record<string, string>>,
   ): Promise<{ filePath: string; entity: EntityFile } | null> {
+    assertSafeEntityId(name);
     const canonical = resolveHistoricalEntityCanonicalId(name, historicalIds);
     for (const candidate of canonical === name ? [name] : [canonical, name]) {
       const filePath = path.join(this.deps.entitiesDir, `${candidate}.md`);
@@ -143,6 +150,7 @@ export class EntityStore {
     method: string,
     mutate: (entity: EntityFile, historicalIds: Readonly<Record<string, string>>) => boolean,
   ): Promise<boolean> {
+    await this.deps.ensureDirectories();
     const stateDir = path.join(this.deps.baseDir, "state");
     return withEntityCanonicalMutationLock(stateDir, async () => {
       for (let attempt = 0; attempt < 3; attempt += 1) {
@@ -375,83 +383,46 @@ export class EntityStore {
     return normalized;
   }
 
-  private async readEntityFileResolvingRename(firstPath: string, entityId: string) {
-    try {
-      return { id: entityId, content: await this.deps.readStorageSecureFile(firstPath) };
-    } catch (err) {
-      if (err instanceof SecureStoreLockedError) throw err;
-      // ENOENT: renamed away (retry below). ELOOP: planted symlink — skip.
-      if (isErrnoCode(err, "ELOOP")) return null;
-      if (!isErrnoCode(err, "ENOENT")) throw err;
-    }
-    const historicalIds = this.deps.currentHistoricalIds();
-    const canonical = resolveHistoricalEntityCanonicalId(entityId, historicalIds);
-    const candidates = new Set([canonical]);
-    for (const legacy of Object.keys(historicalIds)) {
-      if (resolveHistoricalEntityCanonicalId(legacy, historicalIds) === canonical) candidates.add(legacy);
-    }
-    candidates.delete(entityId);
-    for (const candidate of candidates) {
+  async readEntity(name: string): Promise<string> {
+    const filePath = this.deps.resolveEntityFilePath(name);
+    if (filePath === null) return "";
+    // Physical hit wins. Miss skips rename/body-scan only when amortized
+    // completion attests live fingerprint inputs (not bare journal.complete).
+    const readDirect = async (): Promise<string | undefined> => {
       try {
-        return { id: candidate, content: await this.deps.readStorageSecureFile(
-          path.join(this.deps.entitiesDir, `${candidate}.md`),
-        ) };
+        return await this.deps.readStorageSecureFile(filePath);
       } catch (err) {
         if (err instanceof SecureStoreLockedError) throw err;
-        // ENOENT/ELOOP: gone or a planted symlink — try the next candidate.
-        if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "ELOOP")) continue;
+        if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "ELOOP")) return undefined;
         throw err;
       }
-    }
-    return null;
-  }
-
-  async readEntity(name: string): Promise<string> {
-    const filePath = this.deps.resolveEntityFilePath(name);
-    if (filePath === null) {
-      return "";
-    }
-    // Batch reads retry renames unconditionally (a listed file that ENOENTs
-    // is definitionally a race), but a DIRECT lookup of a legacy id must stop
-    // resolving once the journal says migration completed and its file is gone.
-    if (this.deps.entityCanonicalMigrationComplete()) {
+    };
+    const direct = await readDirect();
+    if (direct !== undefined) return direct;
+    if (await this.migrationCompletion.isCompleted(this.deps)) return "";
+    const renamed = await readEntityFileResolvingRename(this.deps, filePath, name);
+    if (renamed !== null) return renamed.content;
+    // Deferred / fingerprint-stale: match legacy filename by body id.
+    let legacy: string | undefined;
+    for (const file of await this.readEntityDirectoryNames()) {
+      if (!file.endsWith(".md") || !(await this.isContainedRegularEntityFile(file))) continue;
+      let content: string;
       try {
-        return await this.deps.readStorageSecureFile(filePath);
+        content = await this.deps.readStorageSecureFile(path.join(this.deps.entitiesDir, file));
       } catch (err) {
         if (err instanceof SecureStoreLockedError) throw err;
-        // ENOENT: gone. ELOOP: a planted symlink the open refused to follow.
-        if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "ELOOP")) return "";
+        if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "ELOOP")) continue;
         throw err;
       }
+      if (isMalformedEntityRecord(content)) continue;
+      const entity = parseEntityFile(content, this.deps.entitySchemas);
+      if (!entity.name.trim() || !entity.type.trim()) continue;
+      if (this.deps.normalizeEntityName(entity.name, entity.type) !== name) continue;
+      if (legacy !== undefined) throw new Error(`Ambiguous legacy entity id: ${name}`);
+      legacy = content;
     }
-    return (await this.readEntityFileResolvingRename(filePath, name))?.content ?? "";
-  }
-
-  /**
-   * True when `entities/` is a real directory inside this memory root.
-   * `realpath` on a symlinked root would treat the link target as trusted,
-   * so the check uses `lstat` first and refuses the scan when the root
-   * itself is a symlink.
-   */
-  private async isTrustedEntitiesRoot(): Promise<boolean> {
-    let rootStat;
-    try {
-      rootStat = await lstat(this.deps.entitiesDir);
-    } catch (err) {
-      if (isErrnoCode(err, "ENOENT")) return false;
-      throw err;
-    }
-    if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) return false;
-    let memoryRoot: string;
-    let entitiesReal: string;
-    try {
-      memoryRoot = await realpath(this.deps.baseDir);
-      entitiesReal = await realpath(this.deps.entitiesDir);
-    } catch (err) {
-      if (isErrnoCode(err, "ENOENT")) return false;
-      throw err;
-    }
-    return pathIsInside(memoryRoot, entitiesReal);
+    // A peer may have renamed the legacy entry after the directory listing.
+    return (await readDirect()) ?? legacy ?? "";
   }
 
   /**
@@ -460,7 +431,7 @@ export class EntityStore {
    * A symlinked entities root yields an empty list and is not read.
    */
   private async readEntityDirectoryNames(): Promise<string[]> {
-    if (!(await this.isTrustedEntitiesRoot())) return [];
+    if (!(await isTrustedEntitiesRoot(this.deps.baseDir, this.deps.entitiesDir))) return [];
     const entries = await readdir(this.deps.entitiesDir);
     const { kept, ignored } = withoutMacOsReplicationJunk(entries, (name) => name);
     warnIgnoredMacOsReplicationJunk(this.deps.entitiesDir, ignored, (message) => {
@@ -595,6 +566,7 @@ export class EntityStore {
 
   /** Re-serialize every entity file into the compiled-truth timeline shape. */
   async migrateEntityFilesToCompiledTruthTimeline(): Promise<{ total: number; migrated: number }> {
+    await this.deps.ensureDirectories();
     const entityNames = await this.listEntityNames();
     const stateDir = path.join(this.deps.baseDir, "state");
     let migrated = 0;
@@ -605,14 +577,20 @@ export class EntityStore {
       // path bumps only the status version, which never re-triggers the
       // completed migration's reference scan.
       const wrote = await withEntityCanonicalMutationLock(stateDir, async () => {
-        const raw = await this.readEntity(entityName);
+        const filePath = this.deps.resolveEntityFilePath(entityName);
+        if (filePath === null) return false;
+        let raw: string;
+        try {
+          raw = await this.deps.readStorageSecureFile(filePath);
+        } catch (err) {
+          if (err instanceof SecureStoreLockedError || !isErrnoCode(err, "ENOENT")) throw err;
+          return false;
+        }
         if (!raw || isMalformedEntityRecord(raw)) return false;
         const parsed = parseEntityFile(raw, this.deps.entitySchemas);
         if (!parsed.name.trim() || !parsed.type.trim()) return false;
         const serialized = serializeEntityFile(parsed, this.deps.entitySchemas, this.deps.okfConformanceEnabled());
         if (raw.trimEnd() === serialized.trimEnd()) return false;
-        const filePath = this.deps.resolveEntityFilePath(entityName);
-        if (filePath === null) return false;
         await this.deps.writeStorageSecureFile(filePath, serialized);
         this.deps.bumpEntityMutationVersion();
         return true;
@@ -641,6 +619,7 @@ export class EntityStore {
       incrementVersion?: boolean;
     } = {},
   ): Promise<void> {
+    await this.deps.ensureDirectories();
     return withEntityCanonicalMutationLock(
       path.join(this.deps.baseDir, "state"),
       () => this.updateEntitySynthesisUnlocked(name, synthesis, options),
@@ -659,17 +638,9 @@ export class EntityStore {
       incrementVersion?: boolean;
     } = {},
   ): Promise<void> {
-    const filePath = path.join(this.deps.entitiesDir, `${name}.md`);
-    let entity: EntityFile;
-    try {
-      const content = await this.deps.readStorageSecureFile(filePath);
-      entity = parseEntityFile(content, this.deps.entitySchemas);
-    } catch (err) {
-      if (err instanceof SecureStoreLockedError) throw err;
-      if (!isErrnoCode(err, "ENOENT")) throw err;
-      log.debug(`updateEntitySynthesis: entity file ${name}.md not found`);
-      return;
-    }
+    const located = await this.readEntityForMutation(name, "updateEntitySynthesis", this.deps.currentHistoricalIds());
+    if (!located) return;
+    const { filePath, entity } = located;
 
     const updatedAt = options.updatedAt?.trim() || entity.synthesisUpdatedAt?.trim() || undefined;
     const entityUpdatedAt = options.entityUpdatedAt?.trim() || updatedAt || entity.updated || new Date().toISOString();
@@ -721,7 +692,7 @@ export class EntityStore {
       // #3207 semantics): resolve empty rather than following it. Within a
       // trusted root, readdir directly so planted entries reach the secure
       // read instead of being swallowed by the listing guards.
-      if (!(await this.isTrustedEntitiesRoot())) return [];
+      if (!(await isTrustedEntitiesRoot(this.deps.baseDir, this.deps.entitiesDir))) return [];
       const listed = await readdir(this.deps.entitiesDir);
       const { kept: entries, ignored } = withoutMacOsReplicationJunk(listed, (name) => name);
       warnIgnoredMacOsReplicationJunk(this.deps.entitiesDir, ignored, (message) => {
@@ -753,11 +724,11 @@ export class EntityStore {
                 // rename: resolve through historical ids like a direct read.
                 // Still-present symlinks/dirs stay skipped.
                 if (await this.isMissingEntry(entry)) {
-                  return await this.readEntityFileResolvingRename(entryPath, entityId);
+                  return await readEntityFileResolvingRename(this.deps, entryPath, entityId);
                 }
                 return null;
               }
-              return await this.readEntityFileResolvingRename(entryPath, entityId);
+              return await readEntityFileResolvingRename(this.deps, entryPath, entityId);
             } catch (err) {
               // Lock and decrypt failures abort the corpus read (the store
               // is unusable or tampered); anything else isolates to this entry.
@@ -873,6 +844,7 @@ export class EntityStore {
    * Returns count of files merged.
    */
   async mergeFragmentedEntities(): Promise<number> {
+    await this.deps.ensureDirectories();
     return withEntityCanonicalMutationLock(
       path.join(this.deps.baseDir, "state"),
       () => this.mergeFragmentedEntitiesUnlocked(),

package/src/storage/memory-read-store.ts

@@ -41,7 +41,7 @@ export interface MemoryReadStoreDeps {
   readonly artifactsDir: string;
   readonly baseDir: string;
   readonly bufferSurpriseLedgerPath: string;
-  collectActiveMemoryPaths(): Promise<string[]>;
+  collectActiveMemoryPaths(options?: CorpusReadOptions): Promise<string[]>;
   readonly correctionsDir: string;
   ensureDirectories(): Promise<void>;
   readonly entitySchemas: PluginConfig["entitySchemas"] | undefined;
@@ -79,6 +79,7 @@ export interface MemoryReadStoreDeps {
     remainingSlots: number,
     remainingInspectionBudget: number,
     readBatchSize: number,
+    options?: CorpusReadOptions,
   ): Promise<{ memories: MemoryFile[]; filePaths: string[] }>;
   readonly reasoningTracesDir: string;
   resolveTierRootDir(tier: "hot" | "cold"): string;
@@ -408,8 +409,10 @@ export class MemoryReadStore {
     maxMemories?: number;
     batchSize?: number;
     updatedAfter?: Date;
+    abortSignal?: AbortSignal;
   } = {}): Promise<{ memories: MemoryFile[]; filePaths: string[] }> {
-    const allPaths = await this.deps.collectActiveMemoryPaths();
+    const readOptions: CorpusReadOptions = { abortSignal: options.abortSignal };
+    const allPaths = await this.deps.collectActiveMemoryPaths(readOptions);
     const sortedPaths = this.deps.orderWindowPaths(allPaths);
     const maxMemories =
       typeof options.maxMemories === "number" && Number.isFinite(options.maxMemories)
@@ -422,6 +425,8 @@ export class MemoryReadStore {
     const selectedPaths: string[] = [];
 
     for (let i = 0; i < sortedPaths.length; i += normalizedBatchSize) {
+      // Per batch: an abandoned caller must not keep the scan running (#3139).
+      checkCorpusReadAbort(readOptions);
       if (
         maxMemories !== undefined
         && (memories.length >= maxMemories || (maxCandidatePaths !== undefined && selectedPaths.length >= maxCandidatePaths))
@@ -436,7 +441,7 @@ export class MemoryReadStore {
       const remainingInspectionBudget = maxCandidatePaths === undefined ? undefined : Math.max(0, maxCandidatePaths - selectedPaths.length);
       const { memories: batchMemories, filePaths: parsedCandidatePaths } = remainingSlots === undefined
         ? {
-            memories: await this.deps.readParsedMemoriesFromPaths(candidateBatchPaths, normalizedBatchSize),
+            memories: await this.deps.readParsedMemoriesFromPaths(candidateBatchPaths, normalizedBatchSize, readOptions),
             filePaths: candidateBatchPaths,
           }
         : await this.deps.readWindowBoundedBatch(
@@ -444,6 +449,7 @@ export class MemoryReadStore {
             remainingSlots,
             remainingInspectionBudget ?? remainingSlots,
             normalizedBatchSize,
+            readOptions,
           );
       selectedPaths.push(...parsedCandidatePaths);
       for (const memory of batchMemories) {

package/src/storage/windowed-memory-read.ts

@@ -1,5 +1,6 @@
 import { stat } from "node:fs/promises";
 import path from "node:path";
+import { throwIfAborted } from "../abort-error.js";
 import { getCachedMemories } from "../memory-cache.js";
 import { readRawMemoryDocument } from "./memory-frontmatter-metadata.js";
 import type { MemoryFile } from "../types.js";
@@ -11,6 +12,12 @@ export interface WindowedMemoryReadOptions {
   updatedAfter?: Date;
   /** Require candidate paths so malformed files can be reported by governance. */
   includeMalformedPaths?: boolean;
+  /**
+   * Caller cancellation. The windowed scan stops at its next batch boundary
+   * when this aborts, so a cancelled read leaves no work running behind it
+   * (issue #3139).
+   */
+  abortSignal?: AbortSignal;
 }
 
 export interface WindowedMemoryReadResult {
@@ -62,6 +69,7 @@ export async function readWindowedMemories(
           })
         )
       ).filter((memory): memory is MemoryFile => memory !== null);
+      throwIfAborted(options.abortSignal, "windowed memory read aborted");
       const memories = orderWindowMemories(selected, hotCache.baseDir);
       return { memories: rememberMemorySnapshots(memories), filePaths: memories.map((memory) => memory.path) };
     }