@remnic/core 10.0.27 → 10.0.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "10.0.27",
+  "version": "10.0.29",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -3347,7 +3347,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^10.0.27"
+    "@remnic/coding-graph": "^10.0.29"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/local-llm-rate-limit.test.ts

@@ -339,7 +339,10 @@ test("process-wide concurrency cap bounds in-flight requests across clients", as
       const config = createConfig({ localLlmUrl: "http://127.0.0.1:1007", localLlmMaxConcurrency: 2 });
       const clients = [new LocalLlmClient(config), new LocalLlmClient(config), new LocalLlmClient(config)];
       const results = await Promise.all(
-        Array.from({ length: 7 }, (_, i) => clients[i % 3].chatCompletion([{ role: "user", content: `q${i}` }]))
+        Array.from({ length: 7 }, (_, i) => clients[i % 3].chatCompletion(
+          [{ role: "user", content: `q${i}` }],
+          { priority: i % 2 === 0 ? "recall-critical" : "background" }
+        ))
       );
       assert.equal(results.filter((r) => r?.content === "ok").length, 7);
       assert.equal(peak, 2);
@@ -404,3 +407,58 @@ test("waiting for a concurrency slot is bounded by the wait budget", async () =>
     }
   );
 });
+
+test("limiter reserves one finite permit for recall-critical work", async () => {
+  const limiter = new LocalLlmConcurrencyLimiter(2);
+  const background = await limiter.acquire();
+  const secondBackground = await limiter.acquire({ timeoutMs: 0 });
+  try {
+    assert.ok(background);
+    assert.equal(secondBackground, null, "background must not consume the recall permit");
+    const foreground = await limiter.acquire({ highPriority: true, timeoutMs: 0 });
+    assert.ok(foreground, "recall must start without waiting for background completion");
+    foreground();
+    foreground();
+    assert.equal(limiter.inFlight, 1, "release remains idempotent");
+  } finally {
+    secondBackground?.();
+    background?.();
+  }
+});
+
+test("reserved background queue survives aborts and responds to resizing", async () => {
+  const limiter = new LocalLlmConcurrencyLimiter(2);
+  const background = await limiter.acquire();
+  assert.ok(background);
+  const controller = new AbortController();
+  const aborted = limiter.acquire({ signal: controller.signal });
+  controller.abort();
+  const abortedRelease = await aborted;
+  abortedRelease?.();
+  assert.equal(abortedRelease, null);
+  const queued = limiter.acquire();
+  const foreground = await limiter.acquire({ highPriority: true, timeoutMs: 0 });
+  assert.ok(foreground);
+  assert.equal(limiter.queued, 1);
+  limiter.setMax(3);
+  const nextBackground = await queued;
+  assert.ok(nextBackground);
+  assert.equal(limiter.inFlight, 3);
+  nextBackground();
+  background();
+  foreground();
+  assert.equal(limiter.inFlight, 0);
+});
+
+test("one-permit and unlimited limits still admit background work", async () => {
+  for (const max of [1, 0]) {
+    const limiter = new LocalLlmConcurrencyLimiter(max);
+    const first = await limiter.acquire({ timeoutMs: 0 });
+    assert.ok(first);
+    const second = await limiter.acquire({ timeoutMs: 0 });
+    assert.equal(second === null, max === 1);
+    second?.();
+    first();
+    assert.equal(limiter.inFlight, 0);
+  }
+});

package/src/local-llm-rate-limit.ts

@@ -151,9 +151,10 @@ export function computeRateLimitDelayMs(
 
 type Waiter = { highPriority: boolean; grant: () => void };
 
-/** FIFO counting semaphore with a high-priority lane (recall-critical first). */
+/** FIFO semaphore; finite caps above one reserve a permit for recall-critical work. */
 export class LocalLlmConcurrencyLimiter {
   private active = 0;
+  private activeBackground = 0;
   private max: number;
   private readonly waiters: Waiter[] = [];
 
@@ -182,20 +183,25 @@ export class LocalLlmConcurrencyLimiter {
     options: { highPriority?: boolean; signal?: AbortSignal; timeoutMs?: number } = {}
   ): Promise<(() => void) | null> {
     const { signal, timeoutMs } = options;
+    const highPriority = options.highPriority === true;
     if (signal?.aborted) return Promise.resolve(null);
-    if (this.max <= 0 || (this.active < this.max && this.waiters.length === 0)) {
+    if (
+      this.canAcquire(highPriority) &&
+      (this.waiters.length === 0 || (highPriority && !this.waiters[0].highPriority))
+    ) {
       this.active += 1;
-      return Promise.resolve(this.releaser());
+      if (!highPriority) this.activeBackground += 1;
+      return Promise.resolve(this.releaser(highPriority));
     }
     if (timeoutMs !== undefined && timeoutMs <= 0) return Promise.resolve(null);
     return new Promise((resolve) => {
       let timer: NodeJS.Timeout | undefined;
       const waiter: Waiter = {
-        highPriority: options.highPriority === true,
+        highPriority,
         grant: () => {
           signal?.removeEventListener("abort", onAbort);
           if (timer) clearTimeout(timer);
-          resolve(this.releaser());
+          resolve(this.releaser(highPriority));
         },
       };
       const onAbort = (): void => {
@@ -217,21 +223,30 @@ export class LocalLlmConcurrencyLimiter {
     });
   }
 
-  private releaser(): () => void {
+  private canAcquire(highPriority: boolean): boolean {
+    return this.max <= 0 || (
+      this.active < this.max &&
+      (highPriority || this.max === 1 || this.activeBackground < this.max - 1)
+    );
+  }
+
+  private releaser(highPriority: boolean): () => void {
     let released = false;
     return () => {
       if (released) return;
       released = true;
       this.active -= 1;
+      if (!highPriority) this.activeBackground -= 1;
       this.pump();
     };
   }
 
   private pump(): void {
-    while (this.waiters.length > 0 && (this.max <= 0 || this.active < this.max)) {
+    while (this.waiters.length > 0 && this.canAcquire(this.waiters[0].highPriority)) {
       const next = this.waiters.shift();
       if (!next) break;
       this.active += 1;
+      if (!next.highPriority) this.activeBackground += 1;
       next.grant();
     }
   }

package/src/orchestration/orchestrator-init.ts

@@ -114,6 +114,7 @@ export class OrchestratorInitCoordinator {
   private startupEpochAbort:
     | { epoch: number; controller: AbortController }
     | undefined;
+  private readonly startupStorageWrites = new Set<Promise<void>>();
 
   constructor(
     private readonly deps: OrchestratorInitDeps,
@@ -127,6 +128,25 @@ export class OrchestratorInitCoordinator {
     this.activeStartupEpoch = null;
   }
 
+  private async initializeStorage(storage: StorageManager): Promise<void> {
+    const write = storage.ensureDirectories({ waitForEntityMigration: false });
+    this.startupStorageWrites.add(write);
+    try {
+      await write;
+    } finally {
+      this.startupStorageWrites.delete(write);
+    }
+  }
+
+  async drainEntityMigrations(): Promise<void> {
+    await Promise.allSettled([...this.startupStorageWrites]);
+    const drains = [this.deps.storage.drainEntityCanonicalIdMigration()];
+    this.deps.storageRouter.forEachCachedStorage((storage) => {
+      drains.push(storage.drainEntityCanonicalIdMigration());
+    });
+    await Promise.all(drains);
+  }
+
   private beginStartupEpoch(): number {
     this.startupEpochAbort?.controller.abort();
     this.startupEpochAbort = undefined;
@@ -305,7 +325,8 @@ export class OrchestratorInitCoordinator {
         logger: (message) => log.info(message),
       });
       await this.deps.storage.loadAliases();
-      await this.deps.storage.ensureDirectories();
+      if (exitIfStale()) return;
+      await this.initializeStorage(this.deps.storage);
       await Promise.all(
         this.deps.config.externalWikis.map((wiki) =>
           assertExternalWikiRootOutsideMemoryDirCanonical(
@@ -323,7 +344,8 @@ export class OrchestratorInitCoordinator {
         for (const ns of namespaces) {
           const sm = await this.deps.storageRouter.storageFor(ns);
           await sm.loadAliases().catch(() => undefined);
-          await sm.ensureDirectories();
+          if (exitIfStale()) return;
+          await this.initializeStorage(sm);
         }
         // Explicitly seed the catalog with all configured namespaces at startup
         // (round 6, cursor Medium — NBLlR). The storageFor loop above fires the
@@ -564,7 +586,8 @@ export class OrchestratorInitCoordinator {
         .catch((err) => {
           log.error(`deferred initialization failed (non-fatal): ${err}`);
         })
-        .finally(() => {
+        .finally(async () => {
+          await this.drainEntityMigrations();
           settleOwnDeferredReady();
         });
     } catch (err) {

package/src/orchestrator.ts

@@ -969,6 +969,7 @@ export class Orchestrator {
     this._orchestratorInitCoordinator?.retireStartupEpoch();
     this.abortDeferredInit();
     this.extractionQueueCoordinator.stopAccepting();
+    await this._orchestratorInitCoordinator?.drainEntityMigrations();
     if (this.wearablesAutoSyncHandle) {
       await this.wearablesAutoSyncHandle.stop();
       this.wearablesAutoSyncHandle = null;

package/src/secure-store/entity-file-read.ts

@@ -0,0 +1,55 @@
+import { constants } from "node:fs";
+import { lstat, open, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+import { assertPathInsideRoot } from "../utils/path-containment.js";
+
+export async function readStoreFileBuffer(filePath: string, memoryDir: string): Promise<Buffer> {
+  const root = path.resolve(memoryDir);
+  const entities = path.join(root, "entities");
+  const target = path.resolve(filePath);
+  const relative = path.relative(entities, target);
+  if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+    return readFile(filePath);
+  }
+  const handle = await open(target, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0));
+  try {
+    for (let parent = path.dirname(target); ; parent = path.dirname(parent)) {
+      const info = await lstat(parent);
+      if (info.isSymbolicLink() || !info.isDirectory()) {
+        throw new Error(`Refusing unsafe entity directory: ${parent}.`);
+      }
+      if (parent === root) break;
+    }
+    const [opened, entry, realRoot, realEntities, realTarget] = await Promise.all([
+      handle.stat(), lstat(target), realpath(root), realpath(entities), realpath(target),
+    ]);
+    if (!opened.isFile() || entry.isSymbolicLink() || !entry.isFile()
+      || opened.dev !== entry.dev || opened.ino !== entry.ino
+      || realEntities !== path.join(realRoot, "entities")) {
+      throw new Error(`Refusing unsafe entity file: ${filePath}.`);
+    }
+    assertPathInsideRoot(realEntities, realTarget, filePath);
+    return await handle.readFile();
+  } finally {
+    await handle.close();
+  }
+}
+
+
+/**
+ * Atomic-replacement-tolerant variant: entity writes ship as temp-file +
+ * rename, so an open/lstat identity mismatch during a concurrent write is an
+ * ordinary race, not tampering. Retry a bounded number of times with fresh
+ * containment checks before surfacing the mismatch as unsafe.
+ */
+export async function readStoreFileBufferTolerant(filePath: string, memoryDir: string, attempts = 3): Promise<Buffer> {
+  for (let attempt = 0; ; attempt += 1) {
+    try {
+      return await readStoreFileBuffer(filePath, memoryDir);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      const identityRace = /unsafe entity (?:file|directory)/.test(message);
+      if (!identityRace || attempt >= attempts - 1) throw err;
+    }
+  }
+}

package/src/secure-store/secure-fs.ts

@@ -61,6 +61,7 @@ import {
 } from "./cipher.js";
 import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
 import { isErrnoCode } from "../utils/errno.js";
+import { readStoreFileBufferTolerant as readStoreFileBuffer } from "./entity-file-read.js";
 
 // ---------------------------------------------------------------------------
 // Error classes
@@ -291,7 +292,7 @@ export async function readMaybeEncryptedFileBuffer(
   memoryDir?: string,
   opts?: { aadFilePath?: string }
 ): Promise<Buffer> {
-  const buf = await readFile(filePath);
+  const buf = memoryDir ? await readStoreFileBuffer(filePath, memoryDir) : await readFile(filePath);
   if (!isEncryptedFile(buf)) {
     // Plain file — legacy or unencrypted store.
     return buf;

package/src/storage/entity-canonical-id-migration-runner.ts

@@ -10,6 +10,38 @@ export class EntityCanonicalIdMigrationRunner {
     private readonly readFingerprint: () => Promise<string> = async () => "static",
   ) {}
 
+  /**
+   * First half of deferred startup: when waiting, ensure the migration runs;
+   * when deferring, pre-read the entity entries the store serves instead and
+   * let the migration start later (finishBegin detaches it).
+   */
+  public async begin(wait: boolean, entitiesDir: string): Promise<void> {
+    if (wait) {
+      await this.ensure();
+      return;
+    }
+    const { readSafeEntityEntries } = await import("./entity-canonical-id-migration.js");
+    await readSafeEntityEntries(entitiesDir);
+  }
+
+  /**
+   * Second half: return the directory-initialized barrier when waiting, or
+   * detach it with a warning when the caller deferred the migration.
+   */
+  public finishBegin(wait: boolean): Promise<void> {
+    const barrier = this.markDirectoriesInitialized();
+    if (wait) return barrier;
+    // Thrown rejections are Error-or-primitive; the message extractor
+    // handles both without asserting a shape.
+    const describeDeferredFailure = (failure: Error | string): string =>
+      failure instanceof Error ? failure.message : failure;
+    void barrier.catch(describeDeferredFailureAndWarn);
+    function describeDeferredFailureAndWarn(failure: Error | string): void {
+      console.warn(`entity canonical-id migration deferred at startup failed: ${describeDeferredFailure(failure)}`);
+    }
+    return Promise.resolve();
+  }
+
   public async markDirectoriesInitialized(): Promise<void> {
     this.directoriesInitialized = true;
     await this.ensure();
@@ -32,6 +64,10 @@ export class EntityCanonicalIdMigrationRunner {
     return this.ensure();
   }
 
+  public async drain(): Promise<void> {
+    while (this.inFlight) await this.inFlight.catch(() => undefined);
+  }
+
   private async runOrReuse(): Promise<void> {
     let fingerprintAtStart = await this.readFingerprint();
     if (this.migrationComplete && fingerprintAtStart === this.completedFingerprint) return;

package/src/storage/entity-canonical-id-migration.ts

@@ -370,7 +370,7 @@ async function assertEntityContentUnchanged(
   }
 }
 
-async function readSafeEntityEntries(entitiesDir: string): Promise<string[]> {
+export async function readSafeEntityEntries(entitiesDir: string): Promise<string[]> {
   const rootStat = await lstat(entitiesDir);
   if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) {
     throw new Error("Refusing entity canonical-id migration through a symlinked or non-directory entities root.");

package/src/storage/entity-canonical-id-references.ts

@@ -1,3 +1,4 @@
+import { statSync } from "node:fs";
 /**
  * Write-boundary surface of the entity canonical-id migration (issue #2213).
  *
@@ -60,6 +61,13 @@ function parseJournalMappings(raw: string): Readonly<Record<string, string>> {
   return cleaned;
 }
 
+/** Whether the journal says its migration completed. Throws like the mapping parser. */
+function parseJournalComplete(raw: string): boolean {
+  // SAFETY: JSON.parse output is unknown; the shape is validated on access.
+  const parsed = JSON.parse(raw) as { complete?: unknown };
+  return parsed !== null && parsed.complete === true;
+}
+
 type JournalRead =
   | { kind: "ok"; raw: string; key: string }
   | { kind: "missing" }
@@ -183,13 +191,57 @@ export async function appendCanonicalizedTombstone(
  * identity, so the next lookup retries instead of caching an empty table
  * under a valid key.
  */
+
+/** Journal lookup result: mapping table plus its completion flag. */
+export type HistoricalEntityIdsEntry = {
+  mappings: Readonly<Record<string, string>>;
+  complete: boolean;
+};
+
+
+
+/**
+ * Memoized journal-completion view keyed on the journal file's stat identity.
+ * entityList consults completion once per listed entity; without this each
+ * consult re-opens and re-reads the journal (quadratic I/O on large corpora).
+ */
+export function memoizedJournalCompletion(
+  cache: HistoricalEntityCanonicalIdCache,
+  stateDir: () => string,
+): () => boolean {
+  let memo: { key: string; complete: boolean } | undefined;
+  return (): boolean => {
+    let key = "missing";
+    try {
+      const journalPath = path.join(stateDir(), ENTITY_CANONICAL_ID_MIGRATION_FILE);
+      const j = statSync(journalPath);
+      key = `${j.dev}:${j.ino}:${j.mtimeMs}:${j.ctimeMs}:${j.size}`;
+    } catch {
+      key = "missing";
+    }
+    if (memo && memo.key === key) return memo.complete;
+    const entry = cache.getEntry(stateDir());
+    memo = { key, complete: entry.complete };
+    return memo.complete;
+  };
+}
+
 export class HistoricalEntityCanonicalIdCache {
   private mappings: Readonly<Record<string, string>> = {};
+  private complete = false;
   private key: string | null = null;
   private stateDir: string | null = null;
 
   get(stateDir: string): Readonly<Record<string, string>> {
-    const lastKnown = this.stateDir === stateDir ? this.mappings : {};
+    return this.getEntry(stateDir).mappings;
+  }
+
+
+  /** Table plus the journal's completion flag, cached under one identity. */
+  getEntry(stateDir: string): HistoricalEntityIdsEntry {
+    const lastKnownMappings = this.stateDir === stateDir ? this.mappings : {};
+    const lastKnownComplete = this.stateDir === stateDir ? this.complete : false;
+    const lastKnown: HistoricalEntityIdsEntry = { mappings: lastKnownMappings, complete: lastKnownComplete };
     const result = readJournalFileNoFollow(path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE));
     if (result.kind === "refused") {
       // Symlinked/non-regular journal: never follow it, never adopt its
@@ -208,9 +260,11 @@ export class HistoricalEntityCanonicalIdCache {
     const key = result.kind === "missing" ? "missing" : result.key;
     if (key !== this.key || stateDir !== this.stateDir) {
       let table: Readonly<Record<string, string>> = {};
+      let complete = false;
       if (result.kind === "ok") {
         try {
           table = parseJournalMappings(result.raw);
+          complete = parseJournalComplete(result.raw);
         } catch {
           // Parse failed under a VALID identity: do not commit the key —
           // the next lookup retries instead of pinning an empty table.
@@ -218,10 +272,11 @@ export class HistoricalEntityCanonicalIdCache {
         }
       }
       this.mappings = table;
+      this.complete = complete;
       this.key = key;
       this.stateDir = stateDir;
     }
-    return this.mappings;
+    return { mappings: this.mappings, complete: this.complete };
   }
 }