@remnic/core 9.69.87 → 9.69.89

package/src/offline-sync.test.ts

@@ -42,6 +42,15 @@ async function readUtf8(root: string, relPath: string): Promise<string> {
   return readFile(path.join(root, relPath), "utf-8");
 }
 
+async function exists(root: string, relPath: string): Promise<boolean> {
+  try {
+    await readFile(path.join(root, relPath));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 test("offline snapshot captures source-of-truth files and excludes private/internal paths", async () => {
   const root = await tempDir("remnic-offline-snapshot");
   try {
@@ -381,6 +390,84 @@ test("offline sync includes retrieval debug snapshots for full-fidelity offline
   }
 });
 
+test("offline sync defers a whole locally diverged shard generation, preserving extras and other namespaces (#3146)", async () => {
+  const localRoot = await tempDir("remnic-offline-embgen-local");
+  const remoteRoot = await tempDir("remnic-offline-embgen-remote");
+  try {
+    // Local: generation drifted since the shared base + a local-only shard
+    // + an unrelated namespace shard that must stay local-authoritative.
+    await write(localRoot, "facts/a.md", "alpha");
+    await write(localRoot, "state/embeddings/shard-0000.json", JSON.stringify({
+      version: 1, provider: "openai", model: "local-drift", entries: {},
+    }));
+    await write(localRoot, "state/embeddings/shard-0050.json", "local only extra");
+    await write(localRoot, "namespaces/team/state/embeddings/shard-0009.json", "team local");
+
+    // Remote generation: base-unchanged 0000 + new 0001.
+    await write(remoteRoot, "facts/a.md", "alpha");
+    await write(remoteRoot, "state/embeddings/shard-0000.json", JSON.stringify({
+      version: 1, provider: "openai", model: "base", entries: {},
+    }));
+
+    // The shared base is REQUIRED for the base-unchanged shortcut below
+    // (CodeRabbit round 4: supply the base so the branch is really tested).
+    const baseSnapshot = await buildOfflineSyncSnapshot({
+      root: remoteRoot,
+      sourceId: "remote",
+      includeContent: true,
+    });
+    await write(remoteRoot, "state/embeddings/shard-0037.json", JSON.stringify({
+      version: 1, provider: "openai", model: "base", entries: {
+        added: { path: "memories/added.md", vector: [1] },
+      },
+    }));
+    const snapshot = await buildOfflineSyncSnapshot({
+      root: remoteRoot,
+      sourceId: "remote",
+      includeContent: true,
+    });
+    const result = await applyOfflineSyncSnapshot({
+      root: localRoot,
+      snapshot,
+      baseFiles: baseSnapshot.files,
+    });
+
+    // An unchanged remote base must not erase local indexing. Deferral is
+    // generation-wide: neither the incoming addition nor deletion of local
+    // extras may partially replace the preserved generation.
+    assert.equal(JSON.parse(await readUtf8(localRoot, "state/embeddings/shard-0000.json")).model, "local-drift");
+    assert.equal(await exists(localRoot, "state/embeddings/shard-0037.json"), false);
+    assert.equal(await readUtf8(localRoot, "state/embeddings/shard-0050.json"), "local only extra");
+    assert.equal(await readUtf8(localRoot, "namespaces/team/state/embeddings/shard-0009.json"), "team local");
+    assert.equal(result.deleted, 0);
+    assert.equal(result.upserted, 0);
+    assert.ok(result.skipped >= 2);
+  } finally {
+    await rm(localRoot, { recursive: true, force: true });
+    await rm(remoteRoot, { recursive: true, force: true });
+  }
+});
+
+test("offline sync excludes embedding-index transaction leftovers (#3146)", async () => {
+  const root = await tempDir("remnic-offline-emb3146");
+  try {
+    await write(root, "facts/a.md", "alpha");
+    await write(root, "state/embeddings.pre-replace.tmp/shard-0000.json", "old generation");
+    await write(root, "state/embeddings.staging.tmp-123/staged.json", "staged");
+    await write(root, "state/embeddings.json.pre-migration.tmp-456", "demoted legacy");
+    await write(root, "namespaces/team/state/embeddings.pre-replace.tmp/shard-0001.json", "ns old");
+
+    const snapshot = await buildOfflineSyncSnapshot({
+      root,
+      sourceId: "remote",
+      includeContent: true,
+    });
+    assert.deepEqual(snapshot.files.map((file) => file.path), ["facts/a.md"]);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
 test("offline sync push-side default excludes live LCM sqlite but apply-side still accepts it (#1786)", async () => {
   const root = await tempDir("remnic-offline-lcm-sqlite");
   try {
@@ -401,6 +488,16 @@ test("offline sync push-side default excludes live LCM sqlite but apply-side sti
     assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/lcm.sqlite-shm"), true);
     assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/lcm.sqlite-wal"), true);
     assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/last_qmd_recall.json"), true);
+    // Sharded embedding index files (#3146) are remote-authoritative like
+    // the legacy single-file index.
+    assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/embeddings/shard-0003.json"), true);
+    assert.equal(
+      shouldPreferIncomingOfflineRuntimeFile("namespaces/team/state/embeddings/shard-0042.json"),
+      true,
+    );
+    assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/embeddings/other.json"), false);
+    // Shard preference is scoped to the embedding index directory.
+    assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/other/shard-0000.json"), false);
 
     // buildOfflineSyncSnapshotForPaths still rejects explicitly excluded
     // paths so callers cannot bypass the default exclude via path lists.
@@ -480,7 +577,7 @@ test("offline sync includes durable runtime state and excludes only transient sy
     await write(root, "state/buffer-surprise-ledger.jsonl", "surprise");
     await write(root, "state/buffer.json", "buffer");
     await write(root, "state/buffer.json.tmp-123-456", "tmp");
-    await write(root, "state/embeddings.json", "embeddings");
+    await write(root, "state/embeddings.json", '{"version":1,"provider":"openai","model":"m","entries":{}}');
     await write(root, "state/entity-mention-index.json", "entities");
     await write(root, "state/index_tags.json", "tags");
     await write(root, "state/index_time.json", "time");
@@ -3204,3 +3301,69 @@ test("plain-file content chunks carry the full-file sha256 (file-content respons
     await rm(root, { recursive: true, force: true });
   }
 });
+
+test("applyOfflineSyncSnapshot consumes readIncomingFile for records without inline content", async () => {
+  const root = await tempDir("remnic-offline-read-incoming");
+  try {
+    const oldBody = "old incoming payload";
+    const newBody = "brand new incoming payload";
+    const shaOf = (content: string) => createHash("sha256").update(content).digest("hex");
+    await write(root, "facts/a.md", oldBody);
+    const baseEntry = { path: "facts/a.md", sha256: shaOf(oldBody), bytes: Buffer.byteLength(oldBody), mtimeMs: 1 };
+    const incoming = { path: "facts/a.md", sha256: shaOf(newBody), bytes: Buffer.byteLength(newBody), mtimeMs: 2 };
+    const snapshot = {
+      format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
+      schemaVersion: 1,
+      createdAt: new Date().toISOString(),
+      sourceId: "remote",
+      includeTranscripts: true,
+      files: [incoming],
+    };
+    const baseFiles = [baseEntry];
+
+    // (a) The callback supplies the verified incoming buffer; the record
+    // carries no contentBase64 at all.
+    const applied = await applyOfflineSyncSnapshot({
+      root,
+      snapshot,
+      baseFiles,
+      currentFiles: baseFiles,
+      readIncomingFile: async (target) =>
+        target.path === "facts/a.md" && target.sha256 === incoming.sha256 ? Buffer.from(newBody) : null,
+    });
+    assert.equal(applied.upserted, 1);
+    assert.equal(await readUtf8(root, "facts/a.md"), newBody);
+
+    // (b) A callback buffer that does not match the declared digest is
+    // rejected BEFORE any publication; the published file is untouched.
+    await write(root, "facts/a.md", oldBody);
+    await assert.rejects(() =>
+      applyOfflineSyncSnapshot({
+        root,
+        snapshot,
+        baseFiles,
+        currentFiles: baseFiles,
+        readIncomingFile: async () => Buffer.from("tampered payload"),
+      }),
+    );
+    assert.equal(await readUtf8(root, "facts/a.md"), oldBody);
+
+    // (c) null keeps the metadata-only behavior: an unchanged member whose
+    // local hash matches the incoming record needs no content.
+    const unchangedSnapshot = {
+      ...snapshot,
+      files: [{ ...incoming, sha256: shaOf(oldBody), bytes: Buffer.byteLength(oldBody) }],
+    };
+    const skipped = await applyOfflineSyncSnapshot({
+      root,
+      snapshot: unchangedSnapshot,
+      baseFiles,
+      currentFiles: baseFiles,
+      readIncomingFile: async () => null,
+    });
+    assert.equal(skipped.upserted, 0);
+    assert.equal(await readUtf8(root, "facts/a.md"), oldBody);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});