@remnic/core 10.0.49 → 10.0.50

package/src/orchestrator.ts

@@ -746,6 +746,7 @@ export class Orchestrator {
   private _dependencyPropagationDelivery: DependencyPropagationDelivery | undefined;
   private _dependencyPropagationRecovery: Promise<void> | null = null;
   private destroyed = false;
+  private acceptingEntityMigrationMaintenance = true;
 
   private get dependencyPropagationDelivery(): DependencyPropagationDelivery {
     if (this.destroyed) throw new Error("orchestrator has been destroyed");
@@ -967,6 +968,7 @@ export class Orchestrator {
    * commands should call it before returning to let Node exit naturally.
    */
   async destroy(): Promise<void> {
+    this.acceptingEntityMigrationMaintenance = false;
     this._orchestratorInitCoordinator?.retireStartupEpoch();
     this.abortDeferredInit();
     this.extractionQueueCoordinator.stopAccepting();
@@ -1499,14 +1501,6 @@ export class Orchestrator {
         (namespace) => this.namespaceSearchRouter.backendForNamespace(namespace),
         (msg) => log.debug(msg),
       );
-      const defaultNs = this.config.defaultNamespace;
-      this.storage.onMemoryWrite = (filePath) => this.prioritizedEmbedding?.enqueue(filePath, defaultNs);
-      this.storageRouter.onStorageCreated = (storage, namespace) => {
-        storage.onMemoryWrite = (filePath) => this.prioritizedEmbedding?.enqueue(filePath, namespace);
-      };
-      this.storageRouter.forEachCachedStorage((storage, namespace) => {
-        storage.onMemoryWrite = (filePath) => this.prioritizedEmbedding?.enqueue(filePath, namespace);
-      });
     }
     this.maintenanceScheduler = new MaintenanceScheduler({
       config,
@@ -1516,6 +1510,13 @@ export class Orchestrator {
       getStorage: () => this.storage, storageForNamespace: (namespace) => this.storageRouter.storageFor(namespace),
       onActivitySynced: (summary) => { if (!this.config.meetings.enabled) return; void this.getMeetingsService().then((s) => s.requestBuildForActivitySync(summary.ranAt, this.config.activity.timezone, summary.touchedDates)).catch((err) => log.warn(`meetings: activity tail-step build failed to schedule: ${err instanceof Error ? err.message : String(err)}`)); },
     });
+    const bindStorage = (storage: StorageManager, namespace: string) => {
+      storage.onEntityMigrationComplete = () => { if (this.acceptingEntityMigrationMaintenance) this.requestQmdMaintenance(); };
+      if (this.prioritizedEmbedding) storage.onMemoryWrite = (filePath) => this.prioritizedEmbedding?.enqueue(filePath, namespace);
+    };
+    bindStorage(this.storage, this.config.defaultNamespace);
+    this.storageRouter.onStorageCreated = bindStorage;
+    this.storageRouter.forEachCachedStorage(bindStorage);
     // Issue #1526: background extraction queue lives on its own coordinator.
     this.extractionQueueCoordinator = new ExtractionQueueCoordinator();
 

package/src/storage/entity-canonical-id-migration-adapter.ts

@@ -1,3 +1,4 @@
+import { log } from "../logger.js";
 import type { EntityFile, MemoryFile } from "../types.js";
 import {
   getFingerprint,
@@ -30,6 +31,7 @@ export interface EntityCanonicalIdMigrationHost {
   invalidateColdMemoriesCache(): void;
   bumpMemoryStatusVersion(): void;
   getEntityMutationVersion(): number;
+  onEntityMigrationComplete?: () => void;
 }
 
 export async function runLegacyEntityCanonicalIdMigration(
@@ -40,6 +42,9 @@ export async function runLegacyEntityCanonicalIdMigration(
 ): Promise<string | undefined> {
   const readMigrationFingerprint = () =>
     getFingerprint(host.baseDir, host.entitiesDir, () => String(host.getEntityMutationVersion()));
+  // The migration bumps the status version only after a pass that rewrote
+  // references; a no-op replay must not ask search maintenance to run.
+  let rewroteReferences = false;
   const completionFingerprint = await migrateLegacyEntityCanonicalIds({
     stateDir: host.stateDir,
     entitiesDir: host.entitiesDir,
@@ -65,9 +70,19 @@ export async function runLegacyEntityCanonicalIdMigration(
     invalidateKnowledgeIndexCache: host.invalidateKnowledgeIndexCache.bind(host),
     invalidateAllMemoriesCache: host.invalidateAllMemoriesCache.bind(host),
     invalidateColdMemoriesCache: host.invalidateColdMemoriesCache.bind(host),
-    bumpMemoryStatusVersion: host.bumpMemoryStatusVersion.bind(host),
+    bumpMemoryStatusVersion: () => {
+      rewroteReferences = true;
+      host.bumpMemoryStatusVersion();
+    },
     readMigrationFingerprint,
   } satisfies EntityCanonicalIdMigrationDependencies);
+  if (completionFingerprint !== undefined && rewroteReferences) {
+    try {
+      host.onEntityMigrationComplete?.();
+    } catch {
+      log.warn("entity migration: search maintenance scheduling failed");
+    }
+  }
   // An undefined completion fingerprint is a deliberate signal (issue #2213):
   // a reconcile marker landed while the run scanned, and recomputing the
   // fingerprint here would seal that pending request into the runner's

package/src/storage/entity-canonical-id-migration.ts

@@ -47,6 +47,7 @@ const EntityCanonicalIdMigrationStateSchema = z.object({
   version: z.literal(ENTITY_CANONICAL_ID_MIGRATION_VERSION),
   complete: z.boolean(),
   mappings: z.record(z.string().min(1)),
+  completionFingerprint: z.string().optional(),
   /**
    * Pairs the migration refused to resolve, PARKED rather than dropped.
    * Deleting the record would strand the collision: once an operator removes
@@ -370,6 +371,13 @@ async function assertEntityContentUnchanged(
   }
 }
 
+/**
+ * Fail-closed entity-entry validation for the startup path: refuses a
+ * symlinked or non-directory entities root and any symlinked entry, without
+ * taking the migration lock. Exported so `ensureDirectories()` can run it
+ * before deferring the migration (#3139) — the same check
+ * `migrateLegacyEntityCanonicalIds` performs under the lock.
+ */
 export async function readSafeEntityEntries(entitiesDir: string): Promise<string[]> {
   const rootStat = await lstat(entitiesDir);
   if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) {
@@ -887,14 +895,11 @@ export async function migrateLegacyEntityCanonicalIds(
           if (!isErrnoCode(error, "ENOENT")) throw error;
         }
       }
-      let state = await readState(deps);
-      if (!state) {
-        state = {
-          version: ENTITY_CANONICAL_ID_MIGRATION_VERSION,
-          complete: false,
-          mappings: {},
-        };
-      }
+      let state: EntityCanonicalIdMigrationState = await readState(deps) ?? {
+        version: ENTITY_CANONICAL_ID_MIGRATION_VERSION,
+        complete: false,
+        mappings: {},
+      };
       const blocked: BlockedEntityPairs = new Map();
       const scanNotes: EntityScanNotes = { junk: new Set(), malformed: new Set() };
       // Parks for pairs whose files are GONE, tracked separately from
@@ -951,6 +956,10 @@ export async function migrateLegacyEntityCanonicalIds(
         // fingerprint could SEAL a marker the check never saw.
         const completionFingerprint = await deps.readMigrationFingerprint?.();
         if (await fileExists(reconcileMarkerPath)) return undefined;
+        if (completionFingerprint !== undefined && state.completionFingerprint !== completionFingerprint) {
+          state = { ...state, complete: true, completionFingerprint };
+          await publishState(deps, state);
+        }
         return completionFingerprint;
       };
       /**

package/src/storage/entity-canonical-id-references.ts

@@ -16,6 +16,7 @@ import { statSync } from "node:fs";
  *   reconciliation pass instead ({@link requestEntityCanonicalIdReconcile}).
  */
 import path from "node:path";
+import { z } from "zod";
 import {
   closeSync,
   constants,
@@ -112,6 +113,29 @@ function readJournalFileNoFollow(statePath: string): JournalRead {
   }
 }
 
+/** The journal fields completion attestation needs; extra journal fields are ignored. */
+const CompletedJournalAttestationSchema = z.object({
+  version: z.literal(1),
+  complete: z.literal(true),
+  completionFingerprint: z.string(),
+});
+
+/** Skip compatibility scans only when journal completion attests to the live inputs. */
+export async function entityCanonicalIdMigrationCompleted(
+  stateDir: string,
+  readFingerprint: () => Promise<string>,
+): Promise<boolean> {
+  const result = readJournalFileNoFollow(path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE));
+  if (result.kind !== "ok") return false;
+  try {
+    const parsed = CompletedJournalAttestationSchema.safeParse(JSON.parse(result.raw));
+    if (!parsed.success) return false;
+    return parsed.data.completionFingerprint === await readFingerprint();
+  } catch {
+    return false;
+  }
+}
+
 export function loadHistoricalEntityCanonicalIds(stateDir: string): Readonly<Record<string, string>> {
   const result = readJournalFileNoFollow(path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE));
   if (result.kind !== "ok") return {};

package/src/storage/entity-migration-completion-cache.nofollow.test.ts

@@ -0,0 +1,223 @@
+/**
+ * Regression: amortized completion must refuse a symlinked entities root
+ * before getFingerprint → fingerprintEntityEntries readdir.
+ *
+ * inputSig already labels the root "L", but isCompleted still runs the full
+ * fingerprint (external tree walk) before any isTrustedEntitiesRoot-style
+ * guard. A planted complete journal makes that path reachable after a
+ * missing-id ENOENT.
+ *
+ * Instrumentation (deterministic): spawn a plain child (not nested --test —
+ * node skips recursive test runs) with --experimental-test-module-mocks that
+ * wraps node:fs/promises.readdir via mock.module, dynamic-imports the real
+ * EntityMigrationCompletionCache, and records every readdir whose realpath is
+ * the external canary (or whose path is the entities symlink). Assert
+ * externalCount === 0 — not merely completed === false. chmod/EACCES is not
+ * used (false-green on this host).
+ *
+ * Dynamic import in the probe is intentional: mock.module must install before
+ * the cache module loads fs.promises (module-loading boundary).
+ *
+ * Temp roots only. Expected today: RED (externalCount >= 1).
+ * After root nofollow guard: externalCount === 0 and completed === false.
+ */
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import {
+  lstat,
+  mkdir,
+  mkdtemp,
+  readFile,
+  realpath,
+  rm,
+  symlink,
+  writeFile,
+} from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { ENTITY_CANONICAL_ID_MIGRATION_FILE } from "./entity-canonical-id-references.js";
+
+function findRepoRoot(startDir: string): string {
+  let dir = startDir;
+  for (;;) {
+    if (existsSync(path.join(dir, "packages", "remnic-core", "package.json"))) return dir;
+    const parent = path.dirname(dir);
+    if (parent === dir) {
+      throw new Error(`remnic repo root not found from ${startDir}`);
+    }
+    dir = parent;
+  }
+}
+
+const REPO_ROOT = findRepoRoot(path.dirname(fileURLToPath(import.meta.url)));
+const CACHE_MODULE = path.join(
+  REPO_ROOT,
+  "packages",
+  "remnic-core",
+  "src",
+  "storage",
+  "entity-migration-completion-cache.ts",
+);
+
+type ProbeResult = {
+  completed: boolean | null;
+  err: string | null;
+  externalCount: number;
+  seen: Array<{ raw: string; real: string }>;
+  canaryReal: string;
+};
+
+test("isCompleted must not readdir through a symlinked entities root", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-entities-root-nofollow-"));
+  const memoryDir = path.join(root, "memory");
+  const canaryDir = path.join(root, "canary-external");
+  const entitiesDir = path.join(memoryDir, "entities");
+  const stateDir = path.join(memoryDir, "state");
+  const probeFile = path.join(root, "completion-cache-nofollow.probe.mts");
+  const resultFile = path.join(root, "probe-result.json");
+
+  try {
+    await mkdir(stateDir, { recursive: true });
+    await mkdir(path.join(memoryDir, "config"), { recursive: true });
+    await mkdir(canaryDir, { recursive: true });
+    await writeFile(path.join(canaryDir, "outside-only.md"), "canary external entity body\n");
+
+    await symlink(canaryDir, entitiesDir);
+    assert.equal(
+      (await lstat(entitiesDir)).isSymbolicLink(),
+      true,
+      "fixture: entities/ must be a symlink to the external canary directory",
+    );
+    const canaryReal = await realpath(canaryDir);
+
+    // Regular complete journal — completionFingerprint compare invokes getFingerprint.
+    await writeFile(
+      path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE),
+      JSON.stringify({
+        version: 1,
+        complete: true,
+        mappings: {},
+        completionFingerprint: "planted-complete-journal-for-symlink-root-regression",
+      }),
+    );
+
+    // Plain probe script (no nested --test: parent already runs under the test
+    // runner, and node skips recursive test-file execution).
+    // Dynamic import loads cache after mock.module wraps readdir.
+    await writeFile(
+      probeFile,
+      `import { mock } from "node:test";
+import path from "node:path";
+import { writeFile } from "node:fs/promises";
+import { readdir as realReaddir, realpath } from "node:fs/promises";
+
+const canaryReal = ${JSON.stringify(canaryReal)};
+const memoryDir = ${JSON.stringify(memoryDir)};
+const entitiesDir = ${JSON.stringify(entitiesDir)};
+const cacheModule = ${JSON.stringify(CACHE_MODULE)};
+const resultFile = ${JSON.stringify(resultFile)};
+const seen = [];
+
+const fsp = await import("node:fs/promises");
+mock.module("node:fs/promises", {
+  namedExports: {
+    ...fsp,
+    readdir: async (p, ...a) => {
+      const raw = path.resolve(String(p));
+      let real = raw;
+      try { real = await realpath(raw); } catch { /* keep raw */ }
+      seen.push({ raw, real });
+      return realReaddir(p, ...a);
+    },
+  },
+});
+
+// Dynamic import: mock.module must wrap fs.promises before the cache loads it.
+const { EntityMigrationCompletionCache } = await import(cacheModule);
+const cache = new EntityMigrationCompletionCache();
+let completed = null;
+let err = null;
+try {
+  completed = await cache.isCompleted({
+    baseDir: memoryDir,
+    entitiesDir,
+    getEntityMutationVersion: () => 0,
+  });
+} catch (e) {
+  err = e && typeof e === "object" && "code" in e ? String(e.code) : String(e);
+}
+const entitiesResolved = path.resolve(entitiesDir);
+const external = seen.filter((s) => s.real === canaryReal || s.raw === entitiesResolved);
+const payload = {
+  completed,
+  err,
+  seen,
+  externalCount: external.length,
+  canaryReal,
+};
+await writeFile(resultFile, JSON.stringify(payload), "utf8");
+console.log(JSON.stringify(payload));
+mock.reset();
+`,
+    );
+
+    const probe = spawnSync(
+      process.execPath,
+      [
+        "--experimental-test-module-mocks",
+        "--import",
+        "tsx",
+        "--conditions=remnic-source",
+        probeFile,
+      ],
+      {
+        encoding: "utf8",
+        cwd: REPO_ROOT,
+        env: {
+          ...process.env,
+          NODE_PATH: path.join(REPO_ROOT, "node_modules"),
+        },
+        timeout: 60_000,
+      },
+    );
+
+    if (probe.error) {
+      assert.fail(`probe failed to start: ${probe.error.message}`);
+    }
+
+    let result: ProbeResult;
+    try {
+      // SAFETY: the child probe in this file writes `resultFile` as a ProbeResult before exiting.
+      result = JSON.parse(await readFile(resultFile, "utf8")) as ProbeResult;
+    } catch {
+      assert.fail(
+        [
+          "probe did not write result file",
+          `status=${String(probe.status)} signal=${String(probe.signal)}`,
+          `stdout:\n${probe.stdout ?? ""}`,
+          `stderr:\n${probe.stderr ?? ""}`,
+        ].join("\n"),
+      );
+      return;
+    }
+
+    assert.equal(
+      result.externalCount,
+      0,
+      [
+        "isCompleted must not readdir a symlinked entities root (external canary).",
+        `externalCount=${result.externalCount} canaryReal=${result.canaryReal}`,
+        `seen=${JSON.stringify(result.seen)}`,
+        result.err ? `isCompleted err=${result.err}` : `completed=${String(result.completed)}`,
+      ].join(" "),
+    );
+    assert.equal(result.err, null, "untrusted root must fail closed without throwing through external IO");
+    assert.equal(result.completed, false, "untrusted entities root must not report migration complete");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/src/storage/entity-migration-completion-cache.ts

@@ -0,0 +1,153 @@
+/**
+ * Amortized entity canonical-id migration completion verdict for entity reads.
+ *
+ * Key covers every getFingerprint input that is cheap to lstat: entity mutation
+ * version, entities/ dir, config/aliases.json, the journal, and reconcile
+ * markers. Full fingerprint (including entity entry enumeration) runs only on
+ * key miss.
+ *
+ * ponytail: in-place entity *content* edits still skip revalidation.
+ */
+
+import { lstat, realpath } from "node:fs/promises";
+import path from "node:path";
+import { SecureStoreLockedError } from "../secure-store/secure-fs.js";
+import { isErrnoCode } from "../utils/errno.js";
+import { pathIsInside } from "../utils/path-containment.js";
+import { getFingerprint } from "./entity-canonical-id-migration.js";
+import {
+  ENTITY_CANONICAL_ID_MIGRATION_FILE,
+  ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER,
+  ENTITY_CANONICAL_ID_RECONCILE_MARKER,
+  entityCanonicalIdMigrationCompleted,
+  resolveHistoricalEntityCanonicalId,
+} from "./entity-canonical-id-references.js";
+
+export type EntityMigrationCompletionCacheDeps = {
+  readonly baseDir: string;
+  readonly entitiesDir: string;
+  getEntityMutationVersion(): number;
+};
+
+/**
+ * True when `entitiesDir` is a real directory inside `baseDir`.
+ * lstat first so a symlinked root is never realpath'd into trust.
+ */
+export async function isTrustedEntitiesRoot(
+  baseDir: string,
+  entitiesDir: string,
+): Promise<boolean> {
+  let rootStat;
+  try {
+    rootStat = await lstat(entitiesDir);
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT")) return false;
+    throw err;
+  }
+  if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) return false;
+  let memoryRoot: string;
+  let entitiesReal: string;
+  try {
+    memoryRoot = await realpath(baseDir);
+    entitiesReal = await realpath(entitiesDir);
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT")) return false;
+    throw err;
+  }
+  return pathIsInside(memoryRoot, entitiesReal);
+}
+
+export class EntityMigrationCompletionCache {
+  private cache: { version: number; inputSig: string; completed: boolean } | undefined;
+
+  /** lstat identity for one fingerprint input path; nofollow; missing → stable token. */
+  private async inputPathSig(filePath: string): Promise<string> {
+    try {
+      const s = await lstat(filePath);
+      return `${s.dev}:${s.ino}:${s.mtimeMs}:${s.ctimeMs}:${s.size}:${s.isSymbolicLink() ? "L" : s.isDirectory() ? "D" : "F"}`;
+    } catch {
+      return "missing";
+    }
+  }
+
+  /**
+   * Cheap coherence key. entities/ absence is uncacheable (random suffix) so
+   * we never pin a false complete.
+   */
+  private async inputSig(deps: EntityMigrationCompletionCacheDeps): Promise<string> {
+    const stateDir = path.join(deps.baseDir, "state");
+    const entitiesSig = await this.inputPathSig(deps.entitiesDir);
+    if (entitiesSig === "missing") return `entities-missing:${Math.random()}`;
+    const [aliasesSig, journalSig, reconcileSig, consumingSig] = await Promise.all([
+      this.inputPathSig(path.join(deps.baseDir, "config", "aliases.json")),
+      this.inputPathSig(path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE)),
+      this.inputPathSig(path.join(stateDir, ENTITY_CANONICAL_ID_RECONCILE_MARKER)),
+      this.inputPathSig(path.join(stateDir, ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER)),
+    ]);
+    return [entitiesSig, aliasesSig, journalSig, reconcileSig, consumingSig].join("|");
+  }
+
+  /**
+   * True when journal completion attests to live fingerprint inputs.
+   * Untrusted entities roots (symlink / outside base) never reach getFingerprint.
+   */
+  async isCompleted(deps: EntityMigrationCompletionCacheDeps): Promise<boolean> {
+    const version = deps.getEntityMutationVersion();
+    const inputSig = await this.inputSig(deps);
+    const cached = this.cache;
+    if (cached && cached.version === version && cached.inputSig === inputSig) return cached.completed;
+    if (!(await isTrustedEntitiesRoot(deps.baseDir, deps.entitiesDir))) {
+      this.cache = { version, inputSig, completed: false };
+      return false;
+    }
+    const completed = await entityCanonicalIdMigrationCompleted(path.join(deps.baseDir, "state"), () =>
+      getFingerprint(deps.baseDir, deps.entitiesDir, () => String(version)));
+    this.cache = { version, inputSig, completed };
+    return completed;
+  }
+}
+
+export type ReadEntityRenameDeps = {
+  readonly entitiesDir: string;
+  currentHistoricalIds(): Readonly<Record<string, string>>;
+  readStorageSecureFile(filePath: string): Promise<string>;
+};
+
+/** Resolve a physical entity path through historical id rename candidates. */
+export async function readEntityFileResolvingRename(
+  deps: ReadEntityRenameDeps,
+  firstPath: string,
+  entityId: string,
+): Promise<{ id: string; content: string } | null> {
+  try {
+    return { id: entityId, content: await deps.readStorageSecureFile(firstPath) };
+  } catch (err) {
+    if (err instanceof SecureStoreLockedError) throw err;
+    // ENOENT: renamed away (retry below). ELOOP: planted symlink — skip.
+    if (isErrnoCode(err, "ELOOP")) return null;
+    if (!isErrnoCode(err, "ENOENT")) throw err;
+  }
+  const historicalIds = deps.currentHistoricalIds();
+  const canonical = resolveHistoricalEntityCanonicalId(entityId, historicalIds);
+  const candidates = new Set([canonical]);
+  for (const legacy of Object.keys(historicalIds)) {
+    if (resolveHistoricalEntityCanonicalId(legacy, historicalIds) === canonical) candidates.add(legacy);
+  }
+  candidates.delete(entityId);
+  for (const candidate of candidates) {
+    try {
+      return {
+        id: candidate,
+        content: await deps.readStorageSecureFile(
+          path.join(deps.entitiesDir, `${candidate}.md`),
+        ),
+      };
+    } catch (err) {
+      if (err instanceof SecureStoreLockedError) throw err;
+      // ENOENT/ELOOP: gone or a planted symlink — try the next candidate.
+      if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "ELOOP")) continue;
+      throw err;
+    }
+  }
+  return null;
+}