@remnic/core 10.0.48 → 10.0.50

package/src/offline-sync-embedding-generation-marker.test.ts

@@ -0,0 +1,245 @@
+// Issue #3182: the chunk-level pull path must advance the node-local
+// generation marker around direct shard writes; partial chunks and no-ops
+// must not disturb a valid completed stamp. Stat pinning emulates a
+// coarse-tick filesystem so only a marker advance can move the stamp.
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { lstat, mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import {
+  EmbeddingIndexFileStore,
+  type EmbeddingIndexEntry,
+  type EmbeddingIndexFile,
+  shardIndexOf,
+} from "./embedding-index-storage.js";
+import { applyOfflineSyncFileContentChunk } from "./offline-sync.js";
+
+// Pin the shard dir to one whole-millisecond tick, emulating a coarse-tick
+// filesystem where the pre- and post-write directory stats alias (same
+// technique as embedding-fallback-cross-process.test.ts).
+const TICK_SEC = 1_700_000_000.5;
+const MODEL = "text-embedding-3-small";
+
+function shardRelFor(id: string): string {
+  return `state/embeddings/shard-${String(shardIndexOf(id, 64)).padStart(4, "0")}.json`;
+}
+
+function indexFile(entries: Record<string, EmbeddingIndexEntry>): EmbeddingIndexFile {
+  return { version: 1, provider: "openai", model: MODEL, entries };
+}
+
+function shardDirOf(memoryDir: string): string {
+  return path.join(memoryDir, "state", "embeddings");
+}
+
+async function seedPublishedGeneration(memoryDir: string): Promise<EmbeddingIndexFileStore> {
+  const stateDir = path.join(memoryDir, "state");
+  const store = new EmbeddingIndexFileStore(
+    path.join(stateDir, "embeddings.json"),
+    path.join(stateDir, "embeddings"),
+    path.join(stateDir, "embedding-fallback-status.json"),
+  );
+  // Same fixture shape as embedding-index-storage.test.ts: a published
+  // sharded generation with one entry, completed through the store's
+  // two-phase marker.
+  await mkdir(path.join(stateDir, "embeddings"), { recursive: true });
+  await writeFile(
+    path.join(stateDir, "embeddings", shardRelFor("mem-old").split("/").pop()!),
+    JSON.stringify(indexFile({ "mem-old": { vector: [1, 0], path: "facts/old.md" } })),
+  );
+  await store.persist(indexFile({ "mem-old": { vector: [1, 0], path: "facts/old.md" } }), {
+    touchedIds: ["mem-old"],
+    memoryId: "mem-old",
+  });
+  return store;
+}
+
+async function readMarker(memoryDir: string): Promise<string> {
+  return (await readFile(path.join(memoryDir, "state", "embeddings.generation"), "utf-8")).trim();
+}
+
+async function pinShardDirStat(memoryDir: string): Promise<number> {
+  const shardDir = shardDirOf(memoryDir);
+  await utimes(shardDir, TICK_SEC, TICK_SEC);
+  return (await lstat(shardDir)).ino;
+}
+
+async function assertInoUnchanged(memoryDir: string, inoBefore: number): Promise<void> {
+  assert.equal(
+    (await lstat(shardDirOf(memoryDir))).ino,
+    inoBefore,
+    "precondition: the pull rewrote the shard in place (no directory swap)",
+  );
+}
+
+test("a completed offline-sync chunk write to a published shard moves the generation stamp", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-chunk-"));
+  try {
+    const store = await seedPublishedGeneration(memoryDir);
+    const shardRel = shardRelFor("mem-old");
+    const updated = Buffer.from(
+      JSON.stringify(indexFile({ "mem-old": { vector: [0, 1], path: "facts/old.md" } })),
+      "utf-8",
+    );
+
+    const inoBefore = await pinShardDirStat(memoryDir);
+    const markerBefore = await readMarker(memoryDir);
+    const stampBefore = await store.identityStamp();
+    assert.equal(await store.identityStamp(), stampBefore, "precondition: the completed stamp is stable");
+
+    // The chunk-level pull path used by converge pulls and CLI hydrate:
+    // no IO hooks, plain fs final write (writeSafeFileFromUpload).
+    const result = await applyOfflineSyncFileContentChunk({
+      root: memoryDir,
+      sourceId: "remote",
+      path: shardRel,
+      sha256: createHash("sha256").update(updated).digest("hex"),
+      bytes: updated.length,
+      mtimeMs: 1000,
+      offset: 0,
+      content: updated,
+    });
+    assert.equal(result.applied, true, "precondition: the shard write applied");
+
+    // Force the exact stat the stamp reads so only a marker advance can move it.
+    await utimes(shardDirOf(memoryDir), TICK_SEC, TICK_SEC);
+    await assertInoUnchanged(memoryDir, inoBefore);
+
+    assert.notEqual(
+      await readMarker(memoryDir),
+      markerBefore,
+      "a peer pull that published new generation bytes must advance the generation marker",
+    );
+    const stampAfter = await store.identityStamp();
+    assert.notEqual(
+      stampAfter,
+      stampBefore,
+      "a peer pull that changed generation bytes must move the warm-cache stamp even when the directory stat aliases",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
+  }
+});
+
+test("a partial chunk (staging only) must not invalidate a valid completed stamp", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-partial-"));
+  try {
+    const store = await seedPublishedGeneration(memoryDir);
+    const shardRel = shardRelFor("mem-old");
+    const onDiskBefore = await readFile(path.join(memoryDir, ...shardRel.split("/")));
+
+    const updated = Buffer.from(
+      JSON.stringify(indexFile({ "mem-old": { vector: [0, 1], path: "facts/old.md" } })),
+      "utf-8",
+    );
+    const inoBefore = await pinShardDirStat(memoryDir);
+    const markerBefore = await readMarker(memoryDir);
+    const stampBefore = await store.identityStamp();
+
+    const result = await applyOfflineSyncFileContentChunk({
+      root: memoryDir,
+      sourceId: "remote",
+      path: shardRel,
+      sha256: createHash("sha256").update(updated).digest("hex"),
+      bytes: updated.length,
+      mtimeMs: 1000,
+      offset: 0,
+      content: updated.subarray(0, Math.floor(updated.length / 2)),
+    });
+    assert.equal(result.done, false, "precondition: the chunk is partial");
+    assert.deepEqual(
+      await readFile(path.join(memoryDir, ...shardRel.split("/"))),
+      onDiskBefore,
+      "precondition: a partial chunk spools to staging, never to the generation",
+    );
+
+    await utimes(shardDirOf(memoryDir), TICK_SEC, TICK_SEC);
+    await assertInoUnchanged(memoryDir, inoBefore);
+
+    assert.equal(await readMarker(memoryDir), markerBefore, "a staging-only write must not touch the generation marker");
+    assert.equal(
+      await store.identityStamp(),
+      stampBefore,
+      "no generation bytes changed: the completed stamp must stay valid so warm caches keep working",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
+  }
+});
+
+test("a skipped no-op chunk (content already identical) must not invalidate the stamp", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-noop-"));
+  try {
+    const store = await seedPublishedGeneration(memoryDir);
+    const shardRel = shardRelFor("mem-old");
+    const onDisk = await readFile(path.join(memoryDir, ...shardRel.split("/")));
+
+    const inoBefore = await pinShardDirStat(memoryDir);
+    const markerBefore = await readMarker(memoryDir);
+    const stampBefore = await store.identityStamp();
+
+    const result = await applyOfflineSyncFileContentChunk({
+      root: memoryDir,
+      sourceId: "remote",
+      path: shardRel,
+      sha256: createHash("sha256").update(onDisk).digest("hex"),
+      bytes: onDisk.length,
+      mtimeMs: 1000,
+      offset: 0,
+      content: onDisk,
+    });
+    assert.equal(result.skipped, true, "precondition: identical content is a no-op");
+
+    await utimes(shardDirOf(memoryDir), TICK_SEC, TICK_SEC);
+    await assertInoUnchanged(memoryDir, inoBefore);
+
+    assert.equal(await readMarker(memoryDir), markerBefore, "a no-op must not touch the generation marker");
+    assert.equal(await store.identityStamp(), stampBefore, "no generation bytes changed: the stamp stays valid");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
+  }
+});
+
+test("a failed direct shard write leaves the marker in-flight: the stamp never repeats", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-failed-"));
+  try {
+    const store = await seedPublishedGeneration(memoryDir);
+    const shardRel = shardRelFor("mem-old");
+    const updated = Buffer.from(
+      JSON.stringify(indexFile({ "mem-old": { vector: [0, 1], path: "facts/old.md" } })),
+      "utf-8",
+    );
+
+    const stampBefore = await store.identityStamp();
+
+    // Force the final destructive write to fail: the publish rename cannot
+    // replace the shard file with a directory in its place.
+    const shardAbs = path.join(memoryDir, ...shardRel.split("/"));
+    await rm(shardAbs);
+    await mkdir(shardAbs);
+
+    await assert.rejects(
+      applyOfflineSyncFileContentChunk({
+        root: memoryDir,
+        sourceId: "remote",
+        path: shardRel,
+        sha256: createHash("sha256").update(updated).digest("hex"),
+        bytes: updated.length,
+        mtimeMs: 1000,
+        offset: 0,
+        content: updated,
+      }),
+    );
+
+    assert.equal(await readMarker(memoryDir), "in-flight", "a failed publication must leave the in-flight barrier");
+    const probeA = await store.identityStamp();
+    const probeB = await store.identityStamp();
+    assert.notEqual(probeA, probeB, "an in-flight stamp never repeats, so cached probes keep revalidating");
+    assert.notEqual(probeA, stampBefore);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
+  }
+});

package/src/offline-sync-embedding-generation.ts

@@ -45,7 +45,12 @@ import type {
   OfflineSyncFileStagingWriteTarget,
   OfflineSyncFileState,
 } from "./offline-sync.js";
-import { type SafeArchiveRoot, resolveSafeArchiveTarget } from "./transfer/fs-utils.js";
+import { type SafeArchiveRoot, prepareSafeArchiveRoot, resolveSafeArchiveTarget } from "./transfer/fs-utils.js";
+import {
+  GENERATION_MARKER_BASENAME,
+  beginGenerationMarker,
+  completeGenerationMarker,
+} from "./embedding-generation-marker.js";
 
 const EMBEDDING_MARKER_BASENAME = "embeddings.json";
 export const EMBEDDING_SHARD_DIR_BASENAME = "embeddings";
@@ -95,7 +100,12 @@ export function embeddingMarkerDirOf(relPath: string): string | null {
  * `.../embeddings` directory, the legacy marker belongs to the sibling
  * `.../embeddings` directory of its state dir.
  */
-export function embeddingGenerationMembership(relPath: string): { kind: "shard" | "marker"; shardDir: string } | null {
+export interface EmbeddingGenerationMembership {
+  kind: "shard" | "marker";
+  shardDir: string;
+}
+
+export function embeddingGenerationMembership(relPath: string): EmbeddingGenerationMembership | null {
   const shardDir = embeddingShardDirOf(relPath);
   if (shardDir) return { kind: "shard", shardDir };
   const markerDir = embeddingMarkerDirOf(relPath);
@@ -103,6 +113,41 @@ export function embeddingGenerationMembership(relPath: string): { kind: "shard"
   return null;
 }
 
+/**
+ * Advance the node-local generation marker around one out-of-band destructive
+ * write or delete of a generation member (issue #3182); the fence reasserts
+ * lock ownership so a stale-broken lock never publishes a stable marker over
+ * a peer's write. Callers must not already hold the generation lock — the
+ * snapshot removal deletes members under it, which is why converge hooks its
+ * own call sites, not the shared IO adapter.
+ */
+export async function advanceEmbeddingGenerationMarker<T>(
+  rootAbs: string,
+  membership: EmbeddingGenerationMembership | null,
+  write: () => Promise<T>,
+  validate?: () => Promise<void>,
+): Promise<T> {
+  if (!membership) {
+    await validate?.();
+    return write();
+  }
+  const root = await prepareSafeArchiveRoot(rootAbs, "advanceEmbeddingGenerationMarker", "root");
+  const stateDirRel = membership.shardDir.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length);
+  const stateDirAbs = await resolveSafeArchiveTarget(root, stateDirRel);
+  const lockPath = embeddingGenerationLockPath(stateDirAbs);
+  return await withEmbeddingGenerationLock(stateDirAbs, async (section) => {
+    const markerAbs = await resolveSafeArchiveTarget(root, `${stateDirRel}/${GENERATION_MARKER_BASENAME}`);
+    await validate?.();
+    await assertEmbeddingGenerationLockHeld(lockPath, section);
+    await beginGenerationMarker(markerAbs);
+    await assertEmbeddingGenerationLockHeld(lockPath, section);
+    const result = await write();
+    await assertEmbeddingGenerationLockHeld(lockPath, section);
+    await completeGenerationMarker(markerAbs);
+    return result;
+  });
+}
+
 export interface IncomingEmbeddingGenerations {
   /** Generation dirs with at least one incoming shard. */
   shardDirs: Set<string>;

package/src/offline-sync-embedding-lock-loss.test.ts

@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import { syncBuiltinESMExports } from "node:module";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { EmbeddingGenerationLockLostError, embeddingGenerationLockPath } from "./embedding-generation-lock.js";
+import { advanceEmbeddingGenerationMarker, embeddingGenerationMembership } from "./offline-sync-embedding-generation.js";
+
+// Lose ownership exactly after the in-flight rename, before any content write.
+test("a direct embedding mutation rechecks ownership after its in-flight marker", async (t) => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "remnic-embedding-fence-"));
+  const stateDir = path.join(root, "state");
+  const relative = "state/embeddings/shard-0000.json";
+  const target = path.join(root, relative);
+  const marker = path.join(stateDir, "embeddings.generation");
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.writeFile(target, "winning generation");
+  await fs.writeFile(marker, "completed generation");
+  const rename = fs.rename;
+  t.mock.method(fs, "rename", async (source: Parameters<typeof rename>[0], destination: Parameters<typeof rename>[1]) => {
+    await rename(source, destination);
+    if (String(destination) === marker && await fs.readFile(marker, "utf8") === "in-flight") {
+      await fs.unlink(embeddingGenerationLockPath(stateDir));
+      await fs.writeFile(marker, "peer completed generation");
+    }
+  });
+  syncBuiltinESMExports();
+  t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports(); });
+  t.after(() => fs.rm(root, { recursive: true, force: true }));
+  let writes = 0;
+  await assert.rejects(
+    advanceEmbeddingGenerationMarker(root, embeddingGenerationMembership(relative), async () => {
+      writes++;
+      await fs.writeFile(target, "stale owner's generation");
+    }),
+    EmbeddingGenerationLockLostError,
+  );
+  assert.equal(writes, 0, "a lost owner must stop before touching generation bytes");
+  assert.equal(await fs.readFile(target, "utf8"), "winning generation");
+  assert.equal(await fs.readFile(marker, "utf8"), "peer completed generation");
+});

package/src/offline-sync-file-io.ts

@@ -77,9 +77,16 @@ export async function plainFileDigest(filePath: string): Promise<string> {
   return hash.digest("hex");
 }
 
-import { lstat, utimes } from "node:fs/promises";
+import { lstat, realpath, utimes } from "node:fs/promises";
 import { isCensusMtimeMs } from "./census-validation.js";
-import { resolveSafeArchiveTarget, type SafeArchiveRoot } from "./transfer/fs-utils.js";
+import path from "node:path";
+import {
+  fromPosixRelPath,
+  isPathInsideRoot,
+  resolveSafeArchiveTarget,
+  type SafeArchiveRoot,
+  validateArchiveRelativePath,
+} from "./transfer/fs-utils.js";
 
 /** Sync-internal directory spooling partial uploads and coordination state. */
 export const SYNC_INTERNAL_DIR = ".offline-sync";
@@ -124,3 +131,55 @@ export async function setSafeFileMtime(
   await utimes(target, mtime, mtime);
   return true;
 }
+
+/**
+ * True when a regular (non-symlink) file exists at the archive-relative path.
+ * A symlink is not a regular local file and reads as absent, so callers treat
+ * it like any other missing path instead of failing the whole push.
+ */
+export async function offlineSyncRegularFilePresent(
+  root: SafeArchiveRoot,
+  relPath: string,
+): Promise<boolean> {
+  const target = path.resolve(root.abs, fromPosixRelPath(validateArchiveRelativePath(relPath, root.errorPrefix)));
+  if (!isPathInsideRoot(root.abs, target)) {
+    throw new Error(`${root.errorPrefix}: record path escapes target root: ${relPath}`);
+  }
+  const st = await lstat(target).catch((error: NodeJS.ErrnoException) => {
+    // SAFETY: Node fs errors carry `code`; only ENOENT means absent, all other
+    // errno codes must propagate so census uncertainty fails closed.
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (!st || st.isSymbolicLink() || !st.isFile()) return false;
+  // A regular file still has to resolve inside the root (a symlinked parent directory escapes it).
+  await resolveSafeArchiveTarget(root, relPath);
+  // The census skips a symlinked directory, so a file reached only through one is absent here too.
+  return (await realpath(target)) === path.join(root.real, path.relative(root.abs, target));
+}
+
+/** A path the census observed vanished before its digest was read: a retryable race, never a confirmed delete. */
+export class OfflineSyncCensusRaceError extends Error {
+  constructor(readonly relPath: string, cause?: unknown) {
+    super(
+      `offline sync census race: ${relPath} vanished during digest; retry the snapshot`,
+      cause === undefined ? undefined : { cause },
+    );
+    this.name = "OfflineSyncCensusRaceError";
+  }
+}
+
+/** The message the CLI push retry loop matches: it defers the path and rebuilds the census. */
+export function changedWhileBuildingChangesetError(relPath: string, cause?: unknown): Error {
+  return new Error(
+    `offline sync file changed while building changeset: ${relPath}`,
+    cause === undefined ? undefined : { cause },
+  );
+}
+
+/** A `.catch` handler: a census race while building a changeset upsert stays retryable. */
+export function censusRaceAsChangedWhileBuilding(relPath: string): (error: Error) => never {
+  return (error) => {
+    throw error instanceof OfflineSyncCensusRaceError ? changedWhileBuildingChangesetError(relPath, error) : error;
+  };
+}