@remnic/core 9.69.101 → 9.69.103

package/src/entity-mention-epoch.ts

@@ -0,0 +1,120 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { appendFileSync, mkdirSync, statSync } from "node:fs";
+import path from "node:path";
+
+/**
+ * Entity-mention freshness, separate from the hot-memories corpus sentinel.
+ *
+ * Every memory write bumps the corpus version so the hot cache stays coherent.
+ * The entity mention index does not depend on heat, decay, lifecycle, or
+ * lastValidatedAt, so those rewrites must not force a fact-file rescan.
+ * Content, entityRef, and origin changes bump this epoch. Metadata-only
+ * frontmatter rewrites run inside `hold()` and leave it alone.
+ *
+ * The on-disk sentinel is the byte size of `state/.entity-mention-epoch.log`
+ * (same pattern as the other version logs). A failed append adds one to an
+ * in-process extra, and `current` is the file size plus that extra. A later
+ * successful append, including one from another process, grows the file and
+ * moves `current` past those failed bumps. `reset` drops caches that
+ * registered at load (the mention index).
+ */
+const fallbackByDir = new Map<string, { extra: number; disk: number }>();
+const suppression = new AsyncLocalStorage<true>();
+let onReset: (() => void) | null = null;
+
+type MentionFrontmatter = {
+  entityRef?: unknown;
+  origin?: unknown;
+};
+
+function epochFile(dir: string): string {
+  return path.join(path.resolve(dir), "state", ".entity-mention-epoch.log");
+}
+
+function mentionField(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function diskSize(filePath: string): number {
+  try {
+    return statSync(filePath).size;
+  } catch {
+    return 0;
+  }
+}
+
+function localEpoch(key: string, disk: number): { extra: number; disk: number } {
+  const existing = fallbackByDir.get(key);
+  if (existing) return existing;
+  const created = { extra: 0, disk };
+  fallbackByDir.set(key, created);
+  return created;
+}
+
+function bump(dir: string): void {
+  const key = path.resolve(dir);
+  const filePath = epochFile(dir);
+  const before = diskSize(filePath);
+  let appended = false;
+  try {
+    mkdirSync(path.dirname(filePath), { recursive: true });
+    appendFileSync(filePath, "x");
+    appended = true;
+  } catch {
+    // The file can still be stat-able (read-only, or the disk is full). The
+    // in-process counter has to move or this process keeps serving a stale index.
+  }
+  const after = diskSize(filePath);
+  const local = localEpoch(key, before);
+  if (!appended || after <= before) local.extra += 1;
+  noteDisk(local, after);
+}
+
+function noteDisk(local: { extra: number; disk: number }, disk: number): void {
+  if (disk < local.disk) local.extra += local.disk - disk;
+  local.disk = disk;
+}
+
+function current(dir: string): number {
+  const key = path.resolve(dir);
+  const disk = diskSize(epochFile(dir));
+  const local = fallbackByDir.get(key);
+  if (!local) return disk;
+  noteDisk(local, disk);
+  // Failed appends stay ahead of the file. A peer's later successful append
+  // grows `disk` and must not land on the same number as those local failures.
+  return disk + local.extra;
+}
+
+function suppressed(): boolean {
+  return suppression.getStore() === true;
+}
+
+function hold<T>(fn: () => T): T {
+  return suppression.run(true, fn);
+}
+
+function registerReset(fn: () => void): void {
+  onReset = fn;
+}
+
+function reset(): void {
+  onReset?.();
+}
+
+function neutral(before: MentionFrontmatter, after: MentionFrontmatter): boolean {
+  return (
+    mentionField(before.entityRef) === mentionField(after.entityRef) &&
+    mentionField(before.origin) === mentionField(after.origin)
+  );
+}
+
+export const entityMentionEpoch = {
+  bump,
+  current,
+  suppressed,
+  hold,
+  neutral,
+  registerReset,
+  reset,
+};

package/src/entity-mention-index-cache.ts

@@ -0,0 +1,443 @@
+import path from "node:path";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { raceAbort } from "./abort-error.js";
+import { entityMentionEpoch } from "./entity-mention-epoch.js";
+import { log } from "./logger.js";
+
+/**
+ * In-memory entity mention index.
+ *
+ * Scope (namespaces, dirs, secure-store key, alias map, native revision) is
+ * the cache key. Mention epoch and entity-file mutation are identity inside
+ * the slot, not part of the key:
+ *
+ * - metadata-only corpus bumps do not move the epoch, so the next recall hits
+ * - entity-file changes rebuild from entity files before the recall returns,
+ *   reusing memory snippets (no fact scan). A new entity is visible on that
+ *   recall; its memory snippets arrive with the next full reconcile. A new
+ *   canonical id schedules that reconcile. Callers share one scan. One
+ *   caller's abort stops that caller waiting and leaves the scan running
+ *   for the others
+ * - memory create/edit/delete moves the epoch. The recall serves the last
+ *   index immediately and one background reconcile runs, plus at most one
+ *   follow-up if that scan overlaps a later write. A recall during either
+ *   scan does not grant another follow-up. Further rescans wait for the
+ *   next recall. After writes quiesce and `settleEntityMentionIndex`
+ *   finishes, the index matches a full rebuild
+ * - secure-store key, alias map, and native revision stay in the key, so a
+ *   change is a miss and awaits a full rebuild. A locked store never reads
+ *   another key's plaintext. Evicting that older scope also stops its scan
+ *   from publishing the persisted index
+ * - a persisted index is readable per storage root. A process-wide cache
+ *   clear blocks every root until that root's own post-clear build publishes.
+ *   One root publishing does not reopen another root's file
+ */
+
+const MAX_SLOTS = 32;
+const SETTLE_ROUNDS = 8;
+
+export type EntityMentionIdentity = {
+  mentionEpoch: string;
+  entityMutation: string;
+};
+
+type ScopedStorage = {
+  dir: string;
+  hotCacheKeyId?(): string;
+  entityAliases?: Readonly<Record<string, string>> | null;
+  getEntityMutationVersion?(): number;
+};
+
+type Slot = {
+  scopeKey: string;
+  index: unknown;
+  identity: EntityMentionIdentity;
+  token: number;
+  rebuild: Promise<void> | null;
+  entityJob: Promise<void> | null;
+  again: boolean;
+  followUps: number;
+  currentIdentity: () => EntityMentionIdentity;
+  buildFull: (abortSignal?: AbortSignal) => Promise<unknown>;
+};
+
+const slots = new Map<string, Slot>();
+const rebuildsByScope = new Map<string, number>();
+const GENERATION_STRIDE = 1_000_000_000;
+let clearEpoch = 0;
+const localGenerationByDir = new Map<string, number>();
+const publishedGenerationByDir = new Map<string, number>();
+const persistTailByDir = new Map<string, Promise<void>>();
+
+function resolveMentionDir(dir: string): string {
+  return path.resolve(dir);
+}
+
+export function entityMentionCacheGeneration(dir: string): number {
+  const resolved = resolveMentionDir(dir);
+  return clearEpoch * GENERATION_STRIDE + (localGenerationByDir.get(resolved) ?? 0);
+}
+
+export function entityMentionPersistedIndexReadable(dir: string): boolean {
+  const resolved = resolveMentionDir(dir);
+  const local = localGenerationByDir.get(resolved) ?? 0;
+  if (clearEpoch === 0 && local === 0) return true;
+  return publishedGenerationByDir.get(resolved) === entityMentionCacheGeneration(resolved);
+}
+
+function dirsInScope(scopeKey: string): string[] {
+  const nsEnd = scopeKey.indexOf("\u001e");
+  const storageEnd = nsEnd === -1 ? -1 : scopeKey.indexOf("\u001e", nsEnd + 1);
+  if (nsEnd === -1 || storageEnd === -1) return [];
+  return scopeKey
+    .slice(nsEnd + 1, storageEnd)
+    .split("\u001f")
+    .map((part) => part.split("\u001d")[0] ?? "")
+    .filter((dir) => dir.length > 0)
+    .map((dir) => resolveMentionDir(dir));
+}
+
+function persistedMentionIndexFiles(scopeKey: string): string[] {
+  return dirsInScope(scopeKey).map((dir) => path.join(dir, "state", "entity-mention-index.json"));
+}
+
+function bumpScopeDirs(scopeKey: string): void {
+  for (const dir of dirsInScope(scopeKey)) {
+    localGenerationByDir.set(dir, (localGenerationByDir.get(dir) ?? 0) + 1);
+  }
+}
+
+function captureDirGenerations(scopeKey: string): Map<string, number> {
+  const captured = new Map<string, number>();
+  for (const dir of dirsInScope(scopeKey)) captured.set(dir, entityMentionCacheGeneration(dir));
+  return captured;
+}
+
+function generationsCurrent(captured: Map<string, number>): boolean {
+  for (const [dir, generation] of captured) {
+    if (entityMentionCacheGeneration(dir) !== generation) return false;
+  }
+  return true;
+}
+
+function publishDirGenerations(captured: Map<string, number>): void {
+  for (const [dir, generation] of captured) {
+    if (entityMentionCacheGeneration(dir) === generation) publishedGenerationByDir.set(dir, generation);
+  }
+}
+
+export async function writePersistedEntityMentionIndex(
+  dir: string,
+  generation: number,
+  content: string,
+): Promise<void> {
+  const resolved = resolveMentionDir(dir);
+  const previous = persistTailByDir.get(resolved) ?? Promise.resolve();
+  const run = previous.catch(() => undefined).then(async () => {
+    if (entityMentionCacheGeneration(resolved) !== generation) return;
+    const file = path.join(resolved, "state", "entity-mention-index.json");
+    await mkdir(path.dirname(file), { recursive: true });
+    const current = await readFile(file, "utf-8").catch(() => "");
+    if (current === content) return;
+    if (entityMentionCacheGeneration(resolved) !== generation) return;
+    await writeFile(file, content, "utf-8");
+  });
+  persistTailByDir.set(resolved, run.then(() => undefined, () => undefined));
+  await run;
+}
+
+function discardPersistedMentionIndex(scopeKey: string): void {
+  for (const file of persistedMentionIndexFiles(scopeKey)) {
+    void rm(file, { force: true }).catch(() => undefined);
+  }
+}
+
+function throwIfMentionAborted(signal?: AbortSignal): void {
+  if (!signal?.aborted) return;
+  const reason = signal.reason;
+  throw reason instanceof Error ? reason : new Error("entity mention rebuild aborted");
+}
+
+function compareStrings(left: string, right: string): number {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+function deduped(values: readonly string[]): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const value of values) {
+    const trimmed = value.trim();
+    if (trimmed.length === 0 || seen.has(trimmed)) continue;
+    seen.add(trimmed);
+    out.push(trimmed);
+  }
+  return out;
+}
+
+function sameIdentity(left: EntityMentionIdentity, right: EntityMentionIdentity): boolean {
+  return left.mentionEpoch === right.mentionEpoch && left.entityMutation === right.entityMutation;
+}
+
+function noteRebuild(scopeKey: string): void {
+  rebuildsByScope.set(scopeKey, (rebuildsByScope.get(scopeKey) ?? 0) + 1);
+}
+
+function scopeFamily(scopeKey: string): string {
+  const nsEnd = scopeKey.indexOf("\u001e");
+  const storageEnd = nsEnd === -1 ? -1 : scopeKey.indexOf("\u001e", nsEnd + 1);
+  const namespaceKey = nsEnd === -1 ? scopeKey : scopeKey.slice(0, nsEnd);
+  const storageKey = nsEnd === -1 || storageEnd === -1 ? "" : scopeKey.slice(nsEnd + 1, storageEnd);
+  const dirs = storageKey
+    .split("\u001f")
+    .map((part) => part.split("\u001d")[0] ?? "")
+    .join("\u001f");
+  return `${namespaceKey}\u001e${dirs}`;
+}
+
+function forgetScope(scopeKey: string, bumpPersisted = false): void {
+  const slot = slots.get(scopeKey);
+  if (slot) {
+    slot.token += 1;
+    slot.again = false;
+  }
+  if (bumpPersisted) bumpScopeDirs(scopeKey);
+  slots.delete(scopeKey);
+  rebuildsByScope.delete(scopeKey);
+}
+
+function evictOldest(scopeKey: string): void {
+  if (slots.size < MAX_SLOTS || slots.has(scopeKey)) return;
+  const oldest = slots.keys().next().value;
+  if (typeof oldest === "string") forgetScope(oldest);
+}
+
+function evictSameFamily(scopeKey: string): void {
+  const family = scopeFamily(scopeKey);
+  for (const key of [...slots.keys()]) {
+    if (key !== scopeKey && scopeFamily(key) === family) forgetScope(key, true);
+  }
+}
+
+function wantsSnippetReconcile(index: unknown): boolean {
+  return (
+    !!index &&
+    typeof index === "object" &&
+    (index as { pendingSnippetReconcile?: boolean }).pendingSnippetReconcile === true
+  );
+}
+
+function armRebuild(slot: Slot): void {
+  // A recall that arrives while a scan is already running must not grant
+  // another follow-up. The chain stops after the one already armed.
+  if (!slot.rebuild) slot.followUps = 1;
+  startRebuild(slot);
+}
+
+function startRebuild(slot: Slot): void {
+  // A recall that arrives during a scan is not a write. The follow-up is
+  // armed only when the scan's end identity differs from the identity it
+  // started with. Setting `again` here made every concurrent recall pay a
+  // second full corpus scan after a stable rebuild.
+  if (slot.rebuild) return;
+  const token = slot.token;
+  const captured = captureDirGenerations(slot.scopeKey);
+  noteRebuild(slot.scopeKey);
+  const started = slot.currentIdentity();
+  slot.again = false;
+  const handle: { pending: Promise<void> | null } = { pending: null };
+  handle.pending = slot
+    .buildFull(undefined)
+    .then((index) => {
+      if (slot.token !== token || !generationsCurrent(captured)) return;
+      publishDirGenerations(captured);
+      const ended = slot.currentIdentity();
+      slot.index = index;
+      if (sameIdentity(started, ended)) {
+        slot.identity = ended;
+        return;
+      }
+      // Scan overlapped later writes. Keep the start identity so the next
+      // recall stays stale and exactly one follow-up reconcile runs.
+      slot.identity = started;
+      slot.again = true;
+    })
+    .catch((err: unknown) => {
+      const message = err instanceof Error ? err.message : String(err);
+      log.warn(`entity mention index rebuild failed: ${message}`);
+    })
+    .finally(() => {
+      if (slot.rebuild === handle.pending) slot.rebuild = null;
+      // One automatic follow-up per caller-started rebuild. A store that keeps
+      // changing during every scan waits for the next recall (or settle) instead
+      // of chaining full scans with nobody waiting.
+      if (!slot.again || slot.followUps <= 0) return;
+      slot.again = false;
+      slot.followUps -= 1;
+      startRebuild(slot);
+    });
+  slot.rebuild = handle.pending;
+}
+
+export function entityMentionScopeKey(
+  namespaces: readonly string[] | undefined,
+  storages: readonly ScopedStorage[],
+  nativeRevision: string,
+): string {
+  const namespaceKey = deduped(namespaces ?? []).join("\u001f");
+  const storageKey = storages
+    .map((storage) => {
+      const aliases = Object.entries(storage.entityAliases ?? {}).sort(([left], [right]) =>
+        compareStrings(left, right),
+      );
+      return `${path.resolve(storage.dir)}\u001d${storage.hotCacheKeyId?.() ?? ""}\u001d${JSON.stringify(aliases)}`;
+    })
+    .join("\u001f");
+  return `${namespaceKey}\u001e${storageKey}\u001e${nativeRevision}`;
+}
+
+export function entityMentionIdentity(storages: readonly ScopedStorage[]): EntityMentionIdentity {
+  return {
+    mentionEpoch: storages.map((storage) => String(entityMentionEpoch.current(storage.dir))).join("\u001f"),
+    entityMutation: storages.map((storage) => String(storage.getEntityMutationVersion?.() ?? 0)).join("\u001f"),
+  };
+}
+
+export function entityMentionFullRebuildsStarted(scopeKey: string): number {
+  return rebuildsByScope.get(scopeKey) ?? 0;
+}
+
+export function entityMentionIndexScopeKeys(): string[] {
+  return [...slots.keys()];
+}
+
+export function dropEntityMentionIndexCache(scopeKey?: string): void {
+  if (scopeKey === undefined) {
+    clearEpoch += 1;
+    for (const key of [...slots.keys()]) {
+      discardPersistedMentionIndex(key);
+      forgetScope(key);
+    }
+    return;
+  }
+  discardPersistedMentionIndex(scopeKey);
+  forgetScope(scopeKey, true);
+}
+
+export async function settleEntityMentionIndex(scopeKey?: string): Promise<void> {
+  const matches = (slot: Slot) => scopeKey === undefined || slot.scopeKey === scopeKey;
+  for (let round = 0; round < SETTLE_ROUNDS; round += 1) {
+    for (const slot of slots.values()) {
+      if (!matches(slot)) continue;
+      if (!slot.rebuild && slot.again) {
+        slot.again = false;
+        startRebuild(slot);
+      }
+    }
+    const pending = [...slots.values()].filter((slot) => matches(slot) && slot.rebuild).map((slot) => slot.rebuild!);
+    if (pending.length === 0) return;
+    await Promise.all(pending);
+  }
+}
+
+export async function resolveEntityMentionIndex<T>(options: {
+  scopeKey: string;
+  currentIdentity: () => EntityMentionIdentity;
+  buildFull: (abortSignal?: AbortSignal) => Promise<T>;
+  rebuildEntities: (previous: T, abortSignal?: AbortSignal) => Promise<T>;
+  abortSignal?: AbortSignal;
+}): Promise<T> {
+  const live = options.currentIdentity();
+  const existing = slots.get(options.scopeKey);
+  if (!existing) {
+    evictSameFamily(options.scopeKey);
+    evictOldest(options.scopeKey);
+    noteRebuild(options.scopeKey);
+    const captured = captureDirGenerations(options.scopeKey);
+    const started = options.currentIdentity();
+    const index = await options.buildFull(options.abortSignal);
+    // A clear or family eviction during this first scan must not publish it.
+    if (!generationsCurrent(captured)) return index;
+    publishDirGenerations(captured);
+    const ended = options.currentIdentity();
+    evictSameFamily(options.scopeKey);
+    evictOldest(options.scopeKey);
+    const slot: Slot = {
+      scopeKey: options.scopeKey,
+      index,
+      identity: sameIdentity(started, ended) ? ended : started,
+      token: 0,
+      rebuild: null,
+      entityJob: null,
+      again: false,
+      followUps: 0,
+      currentIdentity: options.currentIdentity,
+      buildFull: options.buildFull,
+    };
+    slots.set(options.scopeKey, slot);
+    if (!sameIdentity(started, ended)) armRebuild(slot);
+    return index;
+  }
+  existing.currentIdentity = options.currentIdentity;
+  existing.buildFull = options.buildFull;
+  while (existing.identity.entityMutation !== options.currentIdentity().entityMutation) {
+    throwIfMentionAborted(options.abortSignal);
+    const pending = existing.entityJob;
+    if (pending) {
+      await raceAbort(pending, options.abortSignal, "entity mention rebuild aborted");
+      continue;
+    }
+    const liveNow = options.currentIdentity();
+    const epochBefore = existing.identity.mentionEpoch;
+    const mutationBefore = liveNow.entityMutation;
+    let release = (): void => {};
+    const gate = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    existing.entityJob = gate;
+    existing.token += 1;
+    existing.again = false;
+    const token = existing.token;
+    const captured = captureDirGenerations(existing.scopeKey);
+    // Shared scan. This caller's abort stops its wait and leaves the scan
+    // running. Publish when the scan settles, even if this caller has left.
+    // A family eviction bumps the token and the root generation; this scan
+    // must not install its result or start a follow-up that would republish.
+    const scan = options.rebuildEntities(existing.index as T, undefined);
+    void scan
+      .then(
+        (rebuilt) => {
+          if (existing.token !== token || !generationsCurrent(captured)) return;
+          const now = options.currentIdentity();
+          existing.index = rebuilt;
+          const epochStable = epochBefore === liveNow.mentionEpoch && epochBefore === now.mentionEpoch;
+          const mutationStable = mutationBefore === now.entityMutation;
+          existing.identity = {
+            entityMutation: mutationStable ? now.entityMutation : mutationBefore,
+            mentionEpoch: epochStable ? now.mentionEpoch : epochBefore,
+          };
+          if (existing.identity.mentionEpoch !== now.mentionEpoch || wantsSnippetReconcile(rebuilt)) {
+            armRebuild(existing);
+          }
+        },
+        () => {},
+      )
+      .finally(() => {
+        if (existing.entityJob === gate) existing.entityJob = null;
+        release();
+      });
+    const rebuilt = await raceAbort(scan, options.abortSignal, "entity mention rebuild aborted");
+    throwIfMentionAborted(options.abortSignal);
+    return rebuilt;
+  }
+  if (existing.identity.mentionEpoch !== live.mentionEpoch) {
+    armRebuild(existing);
+    return existing.index as T;
+  }
+  return existing.index as T;
+}
+
+entityMentionEpoch.registerReset(() => {
+  dropEntityMentionIndexCache();
+});