@remnic/core 9.69.102 → 9.69.104

package/src/embedding-index-storage.ts

@@ -14,6 +14,7 @@
  * caching or provider logic (that lives in `embedding-fallback.ts`).
  */
 import path from "node:path";
+import { randomUUID } from "node:crypto";
 import { constants as bufferConstants } from "node:buffer";
 import { lstat, mkdir, mkdtemp, realpath, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { log } from "./logger.js";
@@ -165,6 +166,21 @@ function replacementBackupPath(shardDir: string): string {
   return path.join(path.dirname(shardDir), "embeddings.pre-replace.tmp");
 }
 
+/**
+ * Warm-cache generation marker (`<stateDir>/embeddings.generation`).
+ * Two-phase publication barrier: in-flight before a publication's first
+ * destructive write, fresh unique value after its last write. A stamp is
+ * therefore stable only for a completed generation — readers can never
+ * cache a stamp that stays stable while generation bytes change, and an
+ * in-flight stamp never repeats, so probes cached during a failed
+ * completion keep revalidating (PR #3176 codex P2). Plain fs like the
+ * status file.
+ */
+const GENERATION_MARKER_BASENAME = "embeddings.generation";
+
+/** Reserved in-flight marker value; never equals a completion marker. */
+const GENERATION_MARKER_IN_FLIGHT = "in-flight";
+
 /**
  * Stable FNV-1a shard assignment: an entry always hashes to the same shard,
  * so an update or removal never needs a manifest to find its file.
@@ -469,7 +485,9 @@ export class EmbeddingIndexFileStore {
    * the published position after an interrupted replacement. Returns true
    * when a rollback happened. Only fires in the actual rename gap
    * (published directory absent); callers must invalidate any cached index
-   * view when this returns true.
+   * view when this returns true. A successful rollback finalizes the
+   * generation marker strictly: a failed completion write fails the
+   * recovery instead of leaving every search reloading forever.
    */
   async recoverIfInterrupted(fence?: EmbeddingGenerationFence): Promise<boolean> {
     // Only the rename gap (published directory absent) is recoverable. When
@@ -492,6 +510,7 @@ export class EmbeddingIndexFileStore {
     // The rollback rename is destructive: reassert lock ownership first.
     await fence?.();
     await rename(backupPath, this.shardDir);
+    await this.writeGenerationMarker(randomUUID());
     log.warn(
       `embedding index: recovered interrupted replacement; former generation restored from ${backupPath}`,
     );
@@ -514,6 +533,97 @@ export class EmbeddingIndexFileStore {
     }
   }
 
+  /** Barrier first half: strict, so bytes can never change under a stable stamp. */
+  private async beginGenerationMarker(): Promise<void> {
+    await this.writeGenerationMarker(GENERATION_MARKER_IN_FLIGHT);
+  }
+
+  /**
+   * Barrier second half: written after the last content write. Best-effort:
+   * a failure leaves the in-flight marker, whose stamp identityStamp()
+   * never repeats, so cached probes revalidate on every search until a
+   * publication completes.
+   */
+  private async completeGenerationMarker(): Promise<void> {
+    await this.writeGenerationMarker(randomUUID()).catch((err) => {
+      log.warn(`embedding index: could not write generation completion marker: ${err instanceof Error ? err.message : String(err)}`);
+    });
+  }
+
+  private async writeGenerationMarker(value: string): Promise<void> {
+    await this.writeAtomicFile(
+      path.join(path.dirname(this.shardDir), GENERATION_MARKER_BASENAME),
+      value,
+      { bypassSecureIo: true },
+    );
+  }
+
+  /** Marker value, or null when absent/unreadable (pre-marker stores keep the stat stamp). */
+  private async readGenerationMarker(): Promise<string | null> {
+    try {
+      const raw = await readFile(
+        path.join(path.dirname(this.shardDir), GENERATION_MARKER_BASENAME),
+        "utf-8",
+      );
+      const value = raw.trim();
+      return value.length > 0 ? value : null;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Warm-cache stamp. A completed generation marker alone would freeze the
+   * stamp for a pre-marker writer in a rolling deployment (it publishes
+   * shard/index changes without updating the marker), so the marker is
+   * combined with the watched path's stat: the marker keeps consecutive
+   * publications distinct on coarse-tick filesystems, the stat keeps
+   * legacy writers visible. In-flight never repeats, forcing revalidation
+   * until a publication completes. Stores never published through this
+   * module keep the plain stat stamp — a directory stat alone aliases
+   * consecutive publications on coarse-tick filesystems and across inode
+   * reuse.
+   */
+  async identityStamp(): Promise<string> {
+    const layout = await this.detectLayout();
+    if (layout === "sharded") {
+      const info = await lstat(this.shardDir).catch((err) => {
+        // SAFETY: only the errno code is inspected; any other rejection shape rethrows uninterpreted.
+        if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
+        throw new EmbeddingIndexStorageError(
+          `cannot stat embedding index shard directory ${this.shardDir}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      });
+      if (!info) return "rename-gap";
+      const marker = await this.readGenerationMarker();
+      // An in-flight marker must never produce a repeatable stamp: a probe
+      // that cached one (completion write failed) must revalidate on every
+      // later probe until a publication completes.
+      if (marker) {
+        if (marker === GENERATION_MARKER_IN_FLIGHT) return `gen:${randomUUID()}`;
+        return `gen:${marker}:${info.ino}:${info.mtimeMs}`;
+      }
+      return `shard:${info.ino}:${info.mtimeMs}`;
+    }
+    if (layout === "legacy") {
+      const info = await stat(this.indexPath).catch((err) => {
+        // SAFETY: only the errno code is inspected; any other rejection shape rethrows uninterpreted.
+        if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
+        throw new EmbeddingIndexStorageError(
+          `cannot stat embedding index file ${this.indexPath}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      });
+      if (!info) return "rename-gap";
+      const marker = await this.readGenerationMarker();
+      if (marker) {
+        if (marker === GENERATION_MARKER_IN_FLIGHT) return `gen:${randomUUID()}`;
+        return `gen:${marker}:${info.ino}:${info.mtimeMs}`;
+      }
+      return `legacy:${info.ino}:${info.mtimeMs}`;
+    }
+    return "empty";
+  }
+
   /** Header identity of the authoritative generation, read leniently. */
   async identityFromDisk(): Promise<EmbeddingIndexIdentity | null> {
     const layout = await this.detectLayout();
@@ -751,6 +861,8 @@ export class EmbeddingIndexFileStore {
       // Serialization is done: reassert lock ownership immediately before
       // the destructive live-generation writes.
       await opts.fence?.();
+      await this.beginGenerationMarker();
+      await opts.fence?.();
       for (const payload of payloads) {
         await this.writeAtomicFile(
           path.join(this.shardDir, shardFileName(payload.shardIndex)),
@@ -764,6 +876,7 @@ export class EmbeddingIndexFileStore {
           await rm(path.join(this.shardDir, shardFileName(shardIndex)), { force: true });
         }
       }
+      await this.completeGenerationMarker();
       return;
     }
 
@@ -779,7 +892,10 @@ export class EmbeddingIndexFileStore {
       // Serialized above: reassert lock ownership immediately before the
       // destructive atomic write.
       await opts.fence?.();
+      await this.beginGenerationMarker();
+      await opts.fence?.();
       await this.writeAtomicFile(this.indexPath, whole);
+      await this.completeGenerationMarker();
       return;
     }
 
@@ -920,6 +1036,9 @@ export class EmbeddingIndexFileStore {
     // delay between the cleanup and the rename still allows a stale-break,
     // and publishing now would clobber the peer's write.
     await fence?.();
+    // Publication barrier: in-flight before the renames, final at commit.
+    await this.beginGenerationMarker();
+    await fence?.();
     let demoted = false;
     try {
       await stat(this.shardDir);
@@ -931,9 +1050,19 @@ export class EmbeddingIndexFileStore {
     try {
       await rename(stagingDir, this.shardDir);
     } catch (err) {
-      if (demoted) await rename(backupPath, this.shardDir).catch(() => undefined);
+      if (demoted) {
+        try {
+          await rename(backupPath, this.shardDir);
+          await this.completeGenerationMarker();
+        } catch (rollbackErr) {
+          log.warn(`embedding index: publication rollback did not restore the generation; marker stays in-flight: ${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}`);
+        }
+      } else {
+        await this.completeGenerationMarker();
+      }
       throw err;
     }
+    await this.completeGenerationMarker();
     await rm(backupPath, { recursive: true, force: true }).catch((err) => {
       log.warn(`embedding index: could not remove replacement backup ${backupPath}: ${err}`);
     });

package/src/entity-mention-carry.ts

@@ -0,0 +1,31 @@
+export type CarriedMemorySnippet = {
+  memorySnippets: string[];
+  memorySnippetOrigins?: Array<string | undefined>;
+};
+
+/**
+ * Copy snippets from the previous index onto entities that already existed.
+ * A canonical id with nothing to copy was created after that index, so any
+ * memory that already pointed at it was skipped. The caller schedules one
+ * full reconcile for that case.
+ */
+export function applyCarriedMemorySnippets(
+  entities: Iterable<{
+    canonicalId: string;
+    memorySnippets: string[];
+    memorySnippetOrigins?: Array<string | undefined>;
+  }>,
+  carried: ReadonlyMap<string, CarriedMemorySnippet>,
+): boolean {
+  let missing = false;
+  for (const entry of entities) {
+    const previous = carried.get(entry.canonicalId);
+    if (!previous) {
+      missing = true;
+      continue;
+    }
+    entry.memorySnippets = previous.memorySnippets.slice();
+    if (previous.memorySnippetOrigins) entry.memorySnippetOrigins = previous.memorySnippetOrigins.slice();
+  }
+  return missing;
+}

package/src/entity-mention-epoch.ts

@@ -0,0 +1,120 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { appendFileSync, mkdirSync, statSync } from "node:fs";
+import path from "node:path";
+
+/**
+ * Entity-mention freshness, separate from the hot-memories corpus sentinel.
+ *
+ * Every memory write bumps the corpus version so the hot cache stays coherent.
+ * The entity mention index does not depend on heat, decay, lifecycle, or
+ * lastValidatedAt, so those rewrites must not force a fact-file rescan.
+ * Content, entityRef, and origin changes bump this epoch. Metadata-only
+ * frontmatter rewrites run inside `hold()` and leave it alone.
+ *
+ * The on-disk sentinel is the byte size of `state/.entity-mention-epoch.log`
+ * (same pattern as the other version logs). A failed append adds one to an
+ * in-process extra, and `current` is the file size plus that extra. A later
+ * successful append, including one from another process, grows the file and
+ * moves `current` past those failed bumps. `reset` drops caches that
+ * registered at load (the mention index).
+ */
+const fallbackByDir = new Map<string, { extra: number; disk: number }>();
+const suppression = new AsyncLocalStorage<true>();
+let onReset: (() => void) | null = null;
+
+type MentionFrontmatter = {
+  entityRef?: unknown;
+  origin?: unknown;
+};
+
+function epochFile(dir: string): string {
+  return path.join(path.resolve(dir), "state", ".entity-mention-epoch.log");
+}
+
+function mentionField(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function diskSize(filePath: string): number {
+  try {
+    return statSync(filePath).size;
+  } catch {
+    return 0;
+  }
+}
+
+function localEpoch(key: string, disk: number): { extra: number; disk: number } {
+  const existing = fallbackByDir.get(key);
+  if (existing) return existing;
+  const created = { extra: 0, disk };
+  fallbackByDir.set(key, created);
+  return created;
+}
+
+function bump(dir: string): void {
+  const key = path.resolve(dir);
+  const filePath = epochFile(dir);
+  const before = diskSize(filePath);
+  let appended = false;
+  try {
+    mkdirSync(path.dirname(filePath), { recursive: true });
+    appendFileSync(filePath, "x");
+    appended = true;
+  } catch {
+    // The file can still be stat-able (read-only, or the disk is full). The
+    // in-process counter has to move or this process keeps serving a stale index.
+  }
+  const after = diskSize(filePath);
+  const local = localEpoch(key, before);
+  if (!appended || after <= before) local.extra += 1;
+  noteDisk(local, after);
+}
+
+function noteDisk(local: { extra: number; disk: number }, disk: number): void {
+  if (disk < local.disk) local.extra += local.disk - disk;
+  local.disk = disk;
+}
+
+function current(dir: string): number {
+  const key = path.resolve(dir);
+  const disk = diskSize(epochFile(dir));
+  const local = fallbackByDir.get(key);
+  if (!local) return disk;
+  noteDisk(local, disk);
+  // Failed appends stay ahead of the file. A peer's later successful append
+  // grows `disk` and must not land on the same number as those local failures.
+  return disk + local.extra;
+}
+
+function suppressed(): boolean {
+  return suppression.getStore() === true;
+}
+
+function hold<T>(fn: () => T): T {
+  return suppression.run(true, fn);
+}
+
+function registerReset(fn: () => void): void {
+  onReset = fn;
+}
+
+function reset(): void {
+  onReset?.();
+}
+
+function neutral(before: MentionFrontmatter, after: MentionFrontmatter): boolean {
+  return (
+    mentionField(before.entityRef) === mentionField(after.entityRef) &&
+    mentionField(before.origin) === mentionField(after.origin)
+  );
+}
+
+export const entityMentionEpoch = {
+  bump,
+  current,
+  suppressed,
+  hold,
+  neutral,
+  registerReset,
+  reset,
+};