@remnic/core 9.69.87 → 9.69.89

package/src/offline-sync-embedding-generation.ts

@@ -0,0 +1,1153 @@
+// Embedding-index generation semantics for offline sync (issue #3146, PR
+// #3148). Extracted from offline-sync.ts so the god-file line-count ratchet
+// does not grow when the rules gain behavior. The snapshot walker treats
+// every `state/embeddings/` shard set — or a legacy `state/embeddings.json`
+// marker — as ONE generation, in both directions:
+//
+//   1. Atomic replacement (apply side): an incoming generation is staged
+//      complete, verified byte-for-byte, and published through the store's
+//      single backup-swap state machine. A crash or write failure can never
+//      leave a mix of the local and incoming generation on disk, and a
+//      local-only shard of a replaced generation is removed by the swap, not
+//      merged into the incoming set.
+//   2. Filter-to-omit, never filter-to-delete (push side): when an exclusion
+//      filter removes ANY member of a local generation from a push, the
+//      WHOLE generation is omitted — files and deletion metadata — and the
+//      receiver keeps its copy. A filtered generation must never look
+//      deleted, and a partially filtered snapshot must never look like a
+//      complete generation.
+import { createHash, randomBytes } from "node:crypto";
+import { lstat, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import {
+  type EmbeddingIndexFile,
+  EmbeddingIndexFileStore,
+  type EmbeddingIndexIdentity,
+  type EmbeddingIndexStoreIo,
+  EmbeddingIndexStorageError,
+  parseEmbeddingIndexDocument,
+  serializeEmbeddingShard,
+  shardEntriesForIndex,
+  validateShardMembership,
+} from "./embedding-index-storage.js";
+import {
+  assertEmbeddingGenerationLockHeld,
+  type EmbeddingGenerationFence,
+  embeddingGenerationLockPath,
+  withEmbeddingGenerationLock,
+} from "./embedding-generation-lock.js";
+import type { OfflineSyncExcludeFile, OfflineSyncFileTarget } from "./offline-sync-file-io.js";
+import { EMBEDDING_SHARD_FILE_PATTERN } from "./offline-sync-runtime-state.js";
+import { log } from "./logger.js";
+import type {
+  OfflineSyncFileDeleteTarget,
+  OfflineSyncFileStagingReadTarget,
+  OfflineSyncFileStagingWriteTarget,
+  OfflineSyncFileState,
+} from "./offline-sync.js";
+import { type SafeArchiveRoot, resolveSafeArchiveTarget } from "./transfer/fs-utils.js";
+
+const EMBEDDING_MARKER_BASENAME = "embeddings.json";
+export const EMBEDDING_SHARD_DIR_BASENAME = "embeddings";
+
+/**
+ * Canonical `state` dir prefix ("state" or "namespaces/<ns>/state") for a
+ * path's segments, or null when the path is not canonical runtime state.
+ */
+function canonicalStateDirPrefix(parts: string[]): string | null {
+  if (parts[0] === "state") return "state";
+  if (parts[0] === "namespaces" && parts.length >= 4 && parts[2] === "state") {
+    return `namespaces/${parts[1]}/state`;
+  }
+  return null;
+}
+
+/** The sharded-generation directory when `relPath` is one of its shards. */
+export function embeddingShardDirOf(relPath: string): string | null {
+  const parts = relPath.split("/");
+  if (parts.length < 3) return null;
+  if (!EMBEDDING_SHARD_FILE_PATTERN.test(parts[parts.length - 1] ?? "")) return null;
+  if (parts[parts.length - 2] !== EMBEDDING_SHARD_DIR_BASENAME) return null;
+  if (canonicalStateDirPrefix(parts) === null) return null;
+  return parts.slice(0, -1).join("/");
+}
+
+/**
+ * True when `dirPath` is a canonical embeddings generation dir
+ * (`state/embeddings` or `namespaces/<ns>/state/embeddings`). Encodes the
+ * snapshot-field shape contract without a ReDoS-shaped literal (issue #2439).
+ */
+export function isEmbeddingGenerationDirPath(dirPath: string): boolean {
+  const membership = embeddingGenerationMembership(`${dirPath}/shard-0000.json`);
+  return membership !== null && membership.shardDir === dirPath;
+}
+
+/** The state dir when `relPath` is the legacy single-file marker. */
+export function embeddingMarkerDirOf(relPath: string): string | null {
+  const parts = relPath.split("/");
+  if ((parts[parts.length - 1] ?? "") !== EMBEDDING_MARKER_BASENAME) return null;
+  if (canonicalStateDirPrefix(parts) === null) return null;
+  return parts.slice(0, -1).join("/");
+}
+
+/**
+ * Membership of `relPath` in an embedding generation: shards belong to their
+ * `.../embeddings` directory, the legacy marker belongs to the sibling
+ * `.../embeddings` directory of its state dir.
+ */
+export function embeddingGenerationMembership(relPath: string): { kind: "shard" | "marker"; shardDir: string } | null {
+  const shardDir = embeddingShardDirOf(relPath);
+  if (shardDir) return { kind: "shard", shardDir };
+  const markerDir = embeddingMarkerDirOf(relPath);
+  if (markerDir) return { kind: "marker", shardDir: `${markerDir}/${EMBEDDING_SHARD_DIR_BASENAME}` };
+  return null;
+}
+
+export interface IncomingEmbeddingGenerations {
+  /** Generation dirs with at least one incoming shard. */
+  shardDirs: Set<string>;
+  /** State dirs whose incoming snapshot carries the legacy marker. */
+  legacyMarkerDirs: Set<string>;
+}
+
+/** Classify incoming snapshot paths into embedding generations. */
+export function detectIncomingEmbeddingGenerations(incomingPaths: Iterable<string>): IncomingEmbeddingGenerations {
+  const shardDirs = new Set<string>();
+  const legacyMarkerDirs = new Set<string>();
+  for (const relPath of incomingPaths) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (!membership) continue;
+    if (membership.kind === "shard") shardDirs.add(membership.shardDir);
+    else legacyMarkerDirs.add(membership.shardDir);
+  }
+  return { shardDirs, legacyMarkerDirs };
+}
+
+function shardRelPath(shardDirRel: string, shardIndex: number): string {
+  return `${shardDirRel}/shard-${String(shardIndex).padStart(4, "0")}.json`;
+}
+
+function describeIndexReadFailure(
+  read: { outcome: "absent" } | { outcome: "foreign" } | { outcome: "unreadable"; reason: string }
+): string {
+  if (read.outcome === "unreadable") return read.reason;
+  if (read.outcome === "foreign") return "unrecognized index format";
+  return "generation document is absent";
+}
+
+function sha256Hex(buffer: Buffer): string {
+  return createHash("sha256").update(buffer).digest("hex");
+}
+
+async function pathExists(absPath: string): Promise<boolean> {
+  try {
+    await lstat(absPath);
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+}
+
+export interface EmbeddingGenerationTransactionIo {
+  /** Storage-backed local read: hydrates unchanged shards (metadata-only
+   * pulls skip their content) and decrypts secure-store files. */
+  readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
+  writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
+  readStagingFile?: (target: OfflineSyncFileStagingReadTarget) => Promise<Buffer>;
+  deleteFile?: (target: OfflineSyncFileDeleteTarget) => Promise<void>;
+}
+
+export interface EmbeddingGenerationTransactionInput {
+  root: SafeArchiveRoot;
+  /** The generation dir, e.g. `state/embeddings`. */
+  shardDirRel: string;
+  /** Incoming shard rel paths — the complete incoming sharded generation. */
+  incomingShardPaths: string[];
+  /** Incoming legacy marker (rel path + verified content), when the
+   * incoming generation is legacy-layout. Mutually exclusive with
+   * `incomingShardPaths`. */
+  incomingMarker: { path: string; sha256: string; buffer: Buffer | null } | null;
+  /** True when the incoming snapshot ALSO carries the legacy marker while
+   * shards form the generation (mixed remote): the per-file loop then writes
+   * the marker as the inert remote artifact, so this transaction must NOT
+   * remove a local marker. */
+  incomingMarkerPresent: boolean;
+  /** Incoming shard records (sha256 for local hydration of unchanged
+   * metadata-only pulls). */
+  incomingShardStates: ReadonlyMap<string, OfflineSyncFileState>;
+  /** Verified incoming content by rel path (absent for shards the metadata
+   * hydration skipped because the local hash already matches). */
+  incomingBuffers: ReadonlyMap<string, Buffer>;
+  /**
+   * Shared-base states (pre-census): lets the transaction re-evaluate
+   * divergence against ACTUAL disk contents while holding the mutation
+   * lock, instead of trusting the census taken before the lock
+   * (codex P1: a daemon mutation between census and lock was discarded).
+   */
+  baseStates?: ReadonlyMap<string, { sha256: string }>;
+  /**
+   * Paths this apply is pre-authorized to remove (changeset deletes / remote
+   * tombstones). A base member absent from the under-lock census is
+   * divergence UNLESS it is pre-authorized — the retry after a response loss
+   * must re-apply idempotently instead of failing as diverged (#3150
+   * review). A pre-authorized member still present with a changed digest
+   * defers as before, so a daemon rewrite is never silently swept.
+   */
+  justifiedRemovals?: ReadonlySet<string>;
+  /**
+   * Deletion revisions (mtimeMs by path) to record through the delete hook
+   * AFTER the swap publishes. Swept members bypass the per-file delete path,
+   * so without this their tombstones are lost and a later peer without
+   * shared-base evidence can reintroduce the removed shards (codex P1).
+   */
+  deletionMtimeByPath?: ReadonlyMap<string, number>;
+  /**
+   * True when the incoming set claims to be a FRESH creation but carries no
+   * base evidence (manifested changeset members are all base-less upserts):
+   * a non-empty local generation must match the incoming set exactly or the
+   * apply defers as diverged, because the empty-base divergence skip would
+   * otherwise silently replace it (#3150 review).
+   */
+  freshCreateClaim?: boolean;
+  io: EmbeddingGenerationTransactionIo;
+  now: number;
+}
+
+export interface EmbeddingGenerationTransactionResult {
+  shardDirRel: string;
+  upserted: number;
+  deleted: number;
+  /** nextBase states for files the transaction published. */
+  writtenStates: Map<string, OfflineSyncFileState>;
+  /** nextBase paths the transaction removed (stale shards, demoted marker). */
+  removedPaths: Set<string>;
+  /** Paths the per-file apply loop must skip — all handled here. */
+  handledPaths: Set<string>;
+  /** True when the generation was deferred (not published) because the disk
+   * diverged from the shared base while the lock was held. */
+  deferredLocalDivergence?: boolean;
+}
+
+/**
+ * Content for ONE incoming generation member: the verified incoming buffer,
+ * or the LOCAL file hash-verified against the incoming record (metadata-only
+ * pulls skip content whose local hash already matches — the local bytes ARE
+ * the incoming content). Shared by the shard and legacy-marker branches.
+ */
+async function hydratedIncomingBuffer(
+  io: EmbeddingGenerationTransactionIo,
+  root: SafeArchiveRoot,
+  relPath: string,
+  expectedSha256: string,
+  incoming: Buffer | null | undefined,
+): Promise<Buffer> {
+  if (incoming) return incoming;
+  const filePath = await resolveSafeArchiveTarget(root, relPath);
+  const local = io.readFile
+    ? await io.readFile({ root: root.abs, path: relPath, filePath })
+    : await readFile(filePath);
+  if (sha256Hex(local) !== expectedSha256) {
+    throw new EmbeddingIndexStorageError(
+      `cannot hydrate unchanged embedding index file ${relPath}: local digest does not match the incoming record; local generation preserved`,
+    );
+  }
+  return local;
+}
+
+/**
+ * Apply ONE incoming embedding generation atomically (issue #3148, round 4):
+ * recover an interrupted replacement, validate the complete incoming
+ * generation (legacy markers are converted to the sharded layout so the
+ * one-way layout is preserved), stage it, verify the staged bytes, then
+ * publish through the shared backup-swap. Any failure before the swap leaves
+ * the local generation untouched; a crash in the swap gap is rolled back by
+ * the store's recovery on next use.
+ */
+/**
+ * Enumerate and hash the ACTUAL local generation on disk (legacy marker +
+ * every shard) through the configured secure IO — the fresh in-lock census
+ * the stale pre-lock apply census cannot be trusted for (codex P1).
+ */
+/** Scope the shared vault base to this physical embedding generation. */
+function baseStatesForGeneration(
+  base: ReadonlyMap<string, { sha256: string }> | undefined,
+  shardDirRel: string,
+): Map<string, { sha256: string }> {
+  return new Map([...base ?? []].filter(([relPath]) => embeddingGenerationMembership(relPath)?.shardDir === shardDirRel));
+}
+
+async function censusGenerationState(
+  io: EmbeddingGenerationTransactionIo,
+  root: SafeArchiveRoot,
+  markerRel: string,
+  markerAbs: string,
+  shardDirAbs: string,
+  shardDirRel: string,
+): Promise<Map<string, string>> {
+  const census = new Map<string, string>();
+  const readSha = async (relPath: string, filePath: string) =>
+    sha256Hex(io.readFile ? await io.readFile({ root: root.abs, path: relPath, filePath }) : await readFile(filePath));
+  if (await pathExists(markerAbs)) census.set(markerRel, await readSha(markerRel, markerAbs));
+  const names = await readdir(shardDirAbs).catch((err: NodeJS.ErrnoException) => {
+    if (err.code === "ENOENT") return [] as string[];
+    throw err;
+  });
+  for (const name of names.filter((candidate) => EMBEDDING_SHARD_FILE_PATTERN.test(candidate)).sort()) {
+    const relPath = `${shardDirRel}/${name}`;
+    census.set(relPath, await readSha(relPath, path.join(shardDirAbs, name)));
+  }
+  return census;
+}
+
+/** Incoming member states for the divergence helper (digests as fallback). */
+function incomingGenerationStates(input: EmbeddingGenerationTransactionInput): { path: string; sha256: string }[] {
+  const states = input.incomingShardPaths.map((relPath) => ({
+    path: relPath,
+    sha256: input.incomingShardStates.get(relPath)?.sha256 ?? sha256Hex(input.incomingBuffers.get(relPath) ?? Buffer.alloc(0)),
+  }));
+  if (input.incomingMarker) {
+    states.push({ path: input.incomingMarker.path, sha256: input.incomingMarker.sha256 });
+  }
+  return states;
+}
+
+export async function applyEmbeddingGenerationTransaction(
+  input: EmbeddingGenerationTransactionInput
+): Promise<EmbeddingGenerationTransactionResult> {
+  const { root, shardDirRel, io } = input;
+  const stateDirRel = shardDirRel.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length);
+  const stateDirAbs = await resolveSafeArchiveTarget(root, stateDirRel);
+  return await withEmbeddingGenerationLock(stateDirAbs, async (section) => {
+    const lockPath = embeddingGenerationLockPath(stateDirAbs);
+    const fence: EmbeddingGenerationFence = () => assertEmbeddingGenerationLockHeld(lockPath, section);
+    const markerRel = `${stateDirRel}/${EMBEDDING_MARKER_BASENAME}`;
+    const shardDirAbs = await resolveSafeArchiveTarget(root, shardDirRel);
+    const markerAbs = await resolveSafeArchiveTarget(root, markerRel);
+    const store = new EmbeddingIndexFileStore(
+      markerAbs,
+      shardDirAbs,
+      await resolveSafeArchiveTarget(root, `${stateDirRel}/embedding-fallback-status.json`)
+    );
+
+    // 1. Close a pending rename gap (old generation restored) before staging:
+    //    the swap below must never build on a half-replaced generation. The
+    //    fixed backup path is validated first: a planted symlink (or any
+    //    escape) must fail closed instead of being renamed into the published
+    //    position or removed through. The restored target is re-resolved so a
+    //    symlinked restoration cannot slip past the subsequent IO.
+    const backupRel = `${stateDirRel}/embeddings.pre-replace.tmp`;
+    await resolveSafeArchiveTarget(root, backupRel);
+    if (await store.recoverIfInterrupted(fence)) {
+      await resolveSafeArchiveTarget(root, shardDirRel);
+    }
+
+    // Codex P1: the apply census was taken BEFORE this lock, so it may be
+    // stale. Re-enumerate and hash the ACTUAL local generation through the
+    // secure IO and defer the WHOLE generation when the disk no longer
+    // matches the shared base — a concurrent daemon mutation must never be
+    // silently discarded by the swap.
+    // No shared base (fresh catch-up) = no divergence semantics to protect,
+    // UNLESS the apply claims a fresh creation (manifested base-less
+    // upserts): an existing local generation must then match the incoming
+    // set exactly or defer, instead of being silently replaced.
+    const justifiedRemovals = input.justifiedRemovals ?? new Set<string>();
+    const baseStates = baseStatesForGeneration(input.baseStates, shardDirRel);
+    if (baseStates.size > 0) {
+      const localCensus = await censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel);
+      const baseArr = [...baseStates].map(([relPath, state]) => ({ path: relPath, sha256: state.sha256 }));
+      const censusArr = [...localCensus].map(([relPath, sha256]) => ({ path: relPath, sha256 }));
+      const deferrals = divergedEmbeddingGenerationDeferrals({
+        incomingFiles: incomingGenerationStates(input),
+        // A pre-authorized removal keeps its base entry only while still on
+        // disk (digest auth): present members are compared normally — a
+        // daemon rewrite still defers — while an absent member is already
+        // applied and must not diverge the retry (#3150 review).
+        baseFiles: baseArr.filter((member) => !justifiedRemovals.has(member.path) || localCensus.has(member.path)),
+        currentFiles: censusArr,
+      });
+      const incomingShardPaths = new Set(input.incomingShardPaths);
+      const addedOrRemovedSinceBase =
+        censusArr.some((member) =>
+          !baseStates.has(member.path) &&
+          !justifiedRemovals.has(member.path) &&
+          // An incoming member found in the census is the apply's own
+          // already-published work (response-loss retry), not a local
+          // addition; its bytes were verified above.
+          !incomingShardPaths.has(member.path)) ||
+        baseArr.some((member) => !localCensus.has(member.path) && !justifiedRemovals.has(member.path));
+      if (deferrals.length > 0 || addedOrRemovedSinceBase) {
+        const handled = new Set<string>(input.incomingShardPaths);
+        if (input.incomingMarker) handled.add(input.incomingMarker.path);
+        if (input.incomingMarkerPresent) handled.add(markerRel);
+        for (const relPath of localCensus.keys()) handled.add(relPath);
+        log.warn(
+          `embedding generation ${shardDirRel} deferred: disk diverged from the shared base under the mutation lock (${deferrals.length > 0 ? deferrals.join(", ") : "generation membership changed since base"}); nothing published, nothing removed`,
+        );
+        return {
+          shardDirRel,
+          upserted: 0,
+          deleted: 0,
+          writtenStates: new Map<string, OfflineSyncFileState>(),
+          removedPaths: new Set<string>(),
+          handledPaths: handled,
+          deferredLocalDivergence: true,
+        };
+      }
+    } else if (input.freshCreateClaim) {
+      const localCensus = await censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel);
+      if (localCensus.size > 0) {
+        const incomingArr = incomingGenerationStates(input).sort((left, right) => left.path.localeCompare(right.path));
+        const censusArr = [...localCensus]
+          .map(([relPath, sha256]) => ({ path: relPath, sha256 }))
+          .sort((left, right) => left.path.localeCompare(right.path));
+        const identical =
+          censusArr.length === incomingArr.length &&
+          censusArr.every((member, index) => member.path === incomingArr[index]?.path && member.sha256 === incomingArr[index]?.sha256);
+        if (!identical) {
+          const handled = new Set<string>(input.incomingShardPaths);
+          if (input.incomingMarker) handled.add(input.incomingMarker.path);
+          if (input.incomingMarkerPresent) handled.add(markerRel);
+          for (const relPath of localCensus.keys()) handled.add(relPath);
+          log.warn(
+            `embedding generation ${shardDirRel} deferred: base-less creation does not match the existing local generation under the mutation lock; nothing published, nothing removed`,
+          );
+          return {
+            shardDirRel,
+            upserted: 0,
+            deleted: 0,
+            writtenStates: new Map<string, OfflineSyncFileState>(),
+            removedPaths: new Set<string>(),
+            handledPaths: handled,
+            deferredLocalDivergence: true,
+          };
+        }
+      }
+    }
+
+    const published = new Map<string, Buffer>();
+    let identity: EmbeddingIndexIdentity | null = null;
+  // Divergence re-evaluation happens HERE, under the lock, against real
+  // disk bytes: the apply-side census may already be stale.
+  // 2. Build the complete incoming generation (final rel path → plaintext
+  //    bytes) and validate every document BEFORE touching local state.
+  if (input.incomingMarker) {
+    const markerBuffer = await hydratedIncomingBuffer(
+      io,
+      root,
+      input.incomingMarker.path,
+      input.incomingMarker.sha256,
+      input.incomingMarker.buffer,
+    );
+    const read = parseEmbeddingIndexDocument(markerBuffer.toString("utf-8"));
+    if (read.outcome !== "ok") {
+      throw new EmbeddingIndexStorageError(
+        `refusing incoming embedding index generation ${markerRel}: ${describeIndexReadFailure(read)}; local generation preserved`
+      );
+    }
+    identity = { provider: read.file.provider, model: read.file.model };
+    const groups = shardEntriesForIndex(read.file);
+    for (const [shardIndex, entries] of groups) {
+      if (Object.keys(entries).length === 0) continue;
+      published.set(
+        shardRelPath(shardDirRel, shardIndex),
+        Buffer.from(serializeEmbeddingShard(read.file, shardIndex, entries), "utf-8")
+      );
+    }
+  } else {
+    for (const relPath of input.incomingShardPaths) {
+      // Metadata-only pulls skip content whose local hash already matches
+      // the incoming record: the LOCAL bytes ARE the incoming content, so
+      // read and hash-verify them instead of failing (codex rounds 5-6).
+      const state = input.incomingShardStates.get(relPath);
+      const buffer = await hydratedIncomingBuffer(
+        io,
+        root,
+        relPath,
+        state?.sha256 ?? "",
+        input.incomingBuffers.get(relPath),
+      );
+      const read = parseEmbeddingIndexDocument(buffer.toString("utf-8"));
+      if (read.outcome !== "ok") {
+        throw new EmbeddingIndexStorageError(
+          `refusing incoming embedding index shard ${relPath}: ${describeIndexReadFailure(read)}; local generation preserved`
+        );
+      }
+      validateShardMembership(relPath.split("/").pop() ?? "", read.file);
+      const shardIdentity = { provider: read.file.provider, model: read.file.model };
+      if (!identity) identity = shardIdentity;
+      if (identity.provider !== shardIdentity.provider || identity.model !== shardIdentity.model) {
+        throw new EmbeddingIndexStorageError(
+          `refusing mixed-identity incoming embedding index generation in ${shardDirRel}: ${relPath} is ${shardIdentity.provider}/${shardIdentity.model} but the generation is ${identity.provider}/${identity.model}; local generation preserved`
+        );
+      }
+      published.set(relPath, buffer);
+    }
+    if (published.size === 0) {
+      throw new EmbeddingIndexStorageError(
+        `incoming embedding index generation ${shardDirRel} is empty; local generation preserved`
+      );
+    }
+  }
+
+    const stagingRel = `${stateDirRel}/embeddings.staging.tmp-sync-${process.pid}-${input.now}-${randomBytes(8).toString("hex")}`;
+    const stagingAbs = await resolveSafeArchiveTarget(root, stagingRel);
+    try {
+      await mkdir(stagingAbs, { recursive: true });
+
+      // 3. Stage the complete generation. With storage-backed IO the staged
+      //    ciphertext binds its FINAL canonical path (`aadRelPath`), so after
+      //    the directory swap it decrypts at the published path — a physical
+      //    staging-path AAD would make the published generation unreadable.
+      for (const [relPath, buffer] of published) {
+        const stagedRel = `${stagingRel}/${path.basename(relPath)}`;
+        const stagedAbs = await resolveSafeArchiveTarget(root, stagedRel);
+        const finalAbs = await resolveSafeArchiveTarget(root, relPath);
+        if (io.writeStagingFile) {
+          await io.writeStagingFile({
+            root: root.abs,
+            path: stagedRel,
+            filePath: stagedAbs,
+            content: buffer,
+            finalAadFilePath: finalAbs,
+          });
+        } else {
+          await writeFile(stagedAbs, buffer, { mode: 0o600 });
+        }
+      }
+
+      // 4. Verify the staged bytes against the incoming content before the
+      //    published generation is touched at all.
+      for (const [relPath, buffer] of published) {
+        const stagedRel = `${stagingRel}/${path.basename(relPath)}`;
+        const stagedAbs = await resolveSafeArchiveTarget(root, stagedRel);
+        const staged = io.readStagingFile
+          ? await io.readStagingFile({
+              root: root.abs,
+              path: stagedRel,
+              filePath: stagedAbs,
+              finalAadFilePath: await resolveSafeArchiveTarget(root, relPath),
+            })
+          : await readFile(stagedAbs);
+        if (!staged.equals(buffer)) {
+          throw new EmbeddingIndexStorageError(
+            `staged embedding index file ${stagedRel} does not match the incoming content; local generation preserved`
+          );
+        }
+      }
+
+      // 5. Publish with the ONE swap state machine shared with persist(). The
+      //    old directory moves aside wholesale, so local shards absent from
+      //    the incoming generation are removed by the swap — never merged.
+      const localShardRels = (
+        await readdir(shardDirAbs).catch((err) => {
+          if ((err as NodeJS.ErrnoException).code === "ENOENT") return [] as string[];
+          throw err;
+        })
+      )
+        .filter((name) => EMBEDDING_SHARD_FILE_PATTERN.test(name))
+        .map((name) => `${shardDirRel}/${name}`)
+        .sort();
+      await store.publishSwappedGeneration(stagingAbs, fence);
+
+      const writtenStates = new Map<string, OfflineSyncFileState>();
+      for (const [relPath, buffer] of published) {
+        writtenStates.set(relPath, {
+          path: relPath,
+          sha256: sha256Hex(buffer),
+          bytes: buffer.length,
+          mtimeMs: input.now,
+        });
+      }
+      const removedPaths = new Set<string>(localShardRels.filter((rel) => !published.has(rel)));
+      let deleted = removedPaths.size;
+      let markerCleanupFailed = false;
+      if ((await pathExists(markerAbs)) && !input.incomingMarker && !input.incomingMarkerPresent) {
+        // Publication is already committed. Keep its base advancement even if
+        // marker cleanup fails, retaining the marker in base for a retry.
+        try {
+          if (io.deleteFile) await io.deleteFile({ root: root.abs, path: markerRel, filePath: markerAbs });
+          else await rm(markerAbs, { force: true });
+          removedPaths.add(markerRel);
+          deleted += 1;
+        } catch (err) {
+          markerCleanupFailed = true;
+          log.warn(`embedding generation ${shardDirRel}: marker cleanup deferred after publish: ${err}`);
+        }
+      }
+      // Swept members bypass the per-file delete path, so record their
+      // deletion revisions here — mirroring the whole-generation removal
+      // path — or a later peer without shared-base evidence can reintroduce
+      // the removed shards (codex P1). An empty map still records (with
+      // `now`) when the caller declared a deletion context, so tombstones
+      // whose changeset deletes omit mtimeMs are not lost. Every member
+      // gets an attempt before the failure propagates: the push side then
+      // retries a fully-recorded apply or nothing, and the
+      // justified-removals exemption keeps that retry idempotent.
+      if (input.deletionMtimeByPath !== undefined && io.deleteFile) {
+        let firstTombstoneError: unknown = null;
+        for (const relPath of removedPaths) {
+          try {
+            await io.deleteFile({
+              root: root.abs,
+              path: relPath,
+              filePath: await resolveSafeArchiveTarget(root, relPath),
+              mtimeMs: input.deletionMtimeByPath.get(relPath) ?? input.now,
+            });
+          } catch (err) {
+            firstTombstoneError ??= err;
+            log.warn(`embedding generation ${shardDirRel}: tombstone recording failed for ${relPath}: ${err instanceof Error ? err.message : String(err)}`);
+          }
+        }
+        if (firstTombstoneError) throw firstTombstoneError;
+      }
+      return {
+        shardDirRel,
+        upserted: published.size,
+        deleted,
+        writtenStates,
+        removedPaths,
+        handledPaths: new Set([...published.keys(), ...removedPaths, ...(markerCleanupFailed ? [markerRel] : [])]),
+      };
+    } catch (err) {
+      await rm(stagingAbs, { recursive: true, force: true }).catch(() => undefined);
+      throw err;
+    }
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Push-side whole-generation omission
+// ---------------------------------------------------------------------------
+
+export interface EmbeddingGenerationOmissionOptions {
+  rootAbs: string;
+  userExcludeRegexps?: readonly RegExp[];
+  excludeFile?: OfflineSyncExcludeFile;
+  /** Recorded deletion-revision paths: a tombstoned member counts as a
+   * generation member for omission detection even though it is absent on
+   * disk (its exclusion must omit the WHOLE generation). */
+  tombstonedPaths?: readonly string[];
+  /**
+   * Per-path push exclusion predicate (structural + node-local defaults +
+   * user regexps). Defaults to the built-in default excludes + the supplied
+   * user regexps, matching the walk that consumes the result.
+   */
+  isExcludedRelPath?: (relPath: string) => boolean;
+}
+
+export interface EmbeddingGenerationOmission {
+  /** Existing member paths omitted because their generation is partial. */
+  omittedPaths: Set<string>;
+  /** Generation dirs whose members are omitted (snapshot field value). */
+  omittedDirs: string[];
+}
+
+/**
+ * Compute the push-side whole-generation omission set: when ANY on-disk
+ * member of an embedding generation is excluded by a push filter, EVERY
+ * member of that generation is omitted (issue #3148, round 4).
+ */
+export async function computeOmittedEmbeddingGenerationPaths(
+  options: EmbeddingGenerationOmissionOptions
+): Promise<EmbeddingGenerationOmission> {
+  const omittedPaths = new Set<string>();
+  const omittedDirs: string[] = [];
+  const stateDirs = ["state"];
+  let namespaceNames: string[] = [];
+  try {
+    namespaceNames = (await readdir(path.join(options.rootAbs, "namespaces"), { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name);
+  } catch {
+    // no namespaces
+  }
+  for (const ns of namespaceNames) stateDirs.push(`namespaces/${ns}/state`);
+  const isExcluded = options.isExcludedRelPath ??
+    ((relPath: string) =>
+      matchesOfflineSyncDefaultExclude(relPath) ||
+      Boolean(options.userExcludeRegexps?.some((re) => re.test(relPath))));
+
+  for (const stateDir of stateDirs) {
+    const markerRel = `${stateDir}/${EMBEDDING_MARKER_BASENAME}`;
+    const shardDirRel = `${stateDir}/${EMBEDDING_SHARD_DIR_BASENAME}`;
+    const shardDirAbs = path.join(options.rootAbs, ...shardDirRel.split("/"));
+    const memberRels: string[] = [];
+    if (await pathExists(path.join(options.rootAbs, ...markerRel.split("/")))) {
+      memberRels.push(markerRel);
+    }
+    let shardNames: string[] = [];
+    try {
+      shardNames = (await readdir(shardDirAbs, { withFileTypes: true }))
+        .filter((entry) => entry.isFile() && EMBEDDING_SHARD_FILE_PATTERN.test(entry.name))
+        .map((entry) => entry.name);
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+    }
+    for (const name of shardNames) memberRels.push(`${shardDirRel}/${name}`);
+    // A tombstoned member is part of the generation even though it is
+    // already absent on disk: its exclusion must omit the WHOLE generation,
+    // not just drop its revision individually (codex round 6, XkFf).
+    for (const tombstoned of options.tombstonedPaths ?? []) {
+      const membership = embeddingGenerationMembership(tombstoned);
+      if (membership?.shardDir === shardDirRel && !memberRels.includes(tombstoned)) {
+        memberRels.push(tombstoned);
+      }
+    }
+    if (memberRels.length === 0) continue;
+
+    const excludedMembers: string[] = [];
+    for (const relPath of memberRels) {
+      let excluded = isExcluded(relPath);
+      if (!excluded && options.excludeFile) {
+        excluded = await options.excludeFile({
+          root: options.rootAbs,
+          path: relPath,
+          filePath: path.join(options.rootAbs, ...relPath.split("/")),
+        });
+      }
+      if (excluded) excludedMembers.push(relPath);
+    }
+    if (excludedMembers.length === 0) continue;
+    omittedDirs.push(shardDirRel);
+    omittedPaths.add(shardDirRel); // skip new shards after the omission header
+    for (const relPath of memberRels) omittedPaths.add(relPath);
+  }
+  return { omittedPaths, omittedDirs };
+}
+
+/** Reject an exclusion introduced after the snapshot header was computed. */
+export async function assertEmbeddingGenerationStillIncluded(
+  rootAbs: string, shardDirRel: string, excludeFile: OfflineSyncExcludeFile | undefined,
+  isExcludedRelPath: (relPath: string) => boolean, tombstonedPaths: readonly string[],
+): Promise<void> {
+  const current = await computeOmittedEmbeddingGenerationPaths({ rootAbs, excludeFile, isExcludedRelPath, tombstonedPaths });
+  if (current.omittedDirs.includes(shardDirRel)) {
+    throw new EmbeddingIndexStorageError(
+      `embedding generation ${shardDirRel} became partially excluded during snapshot capture; retry with a fresh omission header`,
+    );
+  }
+}
+
+/** True when a path belongs to a generation the push omitted. */
+export function isPathInOmittedEmbeddingGeneration(relPath: string, omittedDirs: ReadonlySet<string>): boolean {
+  const membership = embeddingGenerationMembership(relPath);
+  return membership !== null && omittedDirs.has(membership.shardDir);
+}
+
+export type { EmbeddingIndexFile };
+
+import { matchesOfflineSyncDefaultExclude } from "./offline-sync-exclude-globs.js";
+import type { OfflineSyncDeletionRevision } from "./offline-sync.js";
+
+export interface PushEmbeddingGenerationState {
+  /** Generation dirs omitted by the push filters (snapshot header value). */
+  omittedDirs: string[];
+  /** All omitted member paths (walk skip set). */
+  omittedPaths: Set<string>;
+  /** Deletions with whole-generation suppression applied. */
+  deletions: OfflineSyncDeletionRevision[];
+}
+
+/**
+ * One-call push-side generation state for streaming snapshot builders: the
+ * whole-generation omission set (from user regexps, excludeFile, and default
+ * excludes) plus the deletion list with omitted-generation revisions
+ * suppressed — including revisions for members that are already absent, so a
+ * filtered generation can never be announced as deleted.
+ */
+export async function resolvePushEmbeddingGenerationState(options: {
+  rootAbs: string;
+  includeTranscripts: boolean;
+  userExcludeRegexps?: readonly RegExp[];
+  excludeFile?: OfflineSyncExcludeFile;
+  deletions: readonly OfflineSyncDeletionRevision[];
+}): Promise<PushEmbeddingGenerationState> {
+  const omission = await computeOmittedEmbeddingGenerationPaths({
+    rootAbs: options.rootAbs,
+    userExcludeRegexps: options.userExcludeRegexps,
+    excludeFile: options.excludeFile,
+    // RAW pre-filter tombstones: a deleted member's exclusion must omit the
+    // whole generation even though the member is already absent on disk.
+    tombstonedPaths: options.deletions.map((deletion) => deletion.path),
+    isExcludedRelPath: (relPath) =>
+      matchesOfflineSyncDefaultExclude(relPath) ||
+      Boolean(options.userExcludeRegexps?.some((re) => re.test(relPath))),
+  });
+  const omittedDirs = new Set(omission.omittedDirs);
+  const deletions = options.deletions.filter((deletion) => {
+    if (matchesOfflineSyncDefaultExclude(deletion.path)) return false;
+    if (options.userExcludeRegexps?.some((re) => re.test(deletion.path))) return false;
+    const membership = embeddingGenerationMembership(deletion.path);
+    if (membership && omittedDirs.has(membership.shardDir)) return false;
+    return true;
+  });
+  return { omittedDirs: omission.omittedDirs, omittedPaths: omission.omittedPaths, deletions };
+}
+
+
+interface StorageBackedIndexIoHost {
+  readOfflineSyncFile(filePath: string, opts?: { aadFilePath?: string }): Promise<Buffer>;
+  writeOfflineSyncStagingFile(
+    filePath: string,
+    content: Buffer,
+    opts?: { aadFilePath?: string },
+  ): Promise<void>;
+}
+
+/**
+ * Daemon wiring for the index store IO backed by a StorageManager:
+ * canonical-path secure reads/writes, so sync-published encrypted shards
+ * decrypt in the daemon and daemon-written shards decrypt on sync. Shard
+ * STAGING writes bind the FINAL published path.
+ */
+export function storageBackedIndexStoreIo(storage: StorageBackedIndexIoHost): EmbeddingIndexStoreIo {
+  return {
+    readUtf8: async (filePath) => (await storage.readOfflineSyncFile(filePath)).toString("utf-8"),
+    writeUtf8: (filePath, contents, opts) =>
+      storage.writeOfflineSyncStagingFile(
+        filePath,
+        Buffer.from(contents, "utf-8"),
+        opts?.finalAadFilePath === undefined ? undefined : { aadFilePath: opts.finalAadFilePath },
+      ),
+  };
+}
+
+export interface EmbeddingGenerationRemovalInput {
+  root: SafeArchiveRoot;
+  /** The generation dir, e.g. `state/embeddings`. */
+  shardDirRel: string;
+  io: Pick<EmbeddingGenerationTransactionIo, "deleteFile" | "readFile">;
+  /** Incoming deletion revisions by path, threaded so each removed member's
+   * tombstone is recorded through the configured delete hook AFTER the
+   * atomic empty publication (round 6: raw removal must not lose them). */
+  deletionMtimeByPath?: ReadonlyMap<string, number>;
+  now: number;
+  /**
+   * Shared-base states: when the live generation diverged from the base
+   * (daemon indexed, added, or removed members after the census), the
+   * removal DEFERS instead of erasing the local work (codex P1).
+   */
+  baseStates?: ReadonlyMap<string, { sha256: string }>;
+}
+
+/**
+ * Remove a LOCAL embedding generation atomically because the remote deleted
+ * it wholesale (every member has a deletion revision): recover first, then
+ * demote the published directory through the store's single swap state
+ * machine and delete the backup — a crash or delete failure can never leave
+ * a partial same-identity generation serving silently missing memories.
+ * The local legacy marker, when present, is removed through the same delete
+ * hook the per-file path uses.
+ */
+export async function applyEmbeddingGenerationRemoval(
+  input: EmbeddingGenerationRemovalInput,
+): Promise<EmbeddingGenerationTransactionResult> {
+  const { root, shardDirRel, io } = input;
+  const stateDirRel = shardDirRel.slice(0, -(`/${EMBEDDING_SHARD_DIR_BASENAME}`.length));
+  const stateDirAbs = await resolveSafeArchiveTarget(root, stateDirRel);
+  return await withEmbeddingGenerationLock(stateDirAbs, async (section) => {
+    const lockPath = embeddingGenerationLockPath(stateDirAbs);
+    const fence: EmbeddingGenerationFence = () => assertEmbeddingGenerationLockHeld(lockPath, section);
+    const markerRel = `${stateDirRel}/${EMBEDDING_MARKER_BASENAME}`;
+    const shardDirAbs = await resolveSafeArchiveTarget(root, shardDirRel);
+    const markerAbs = await resolveSafeArchiveTarget(root, markerRel);
+    const store = new EmbeddingIndexFileStore(
+      markerAbs,
+      shardDirAbs,
+      await resolveSafeArchiveTarget(root, `${stateDirRel}/embedding-fallback-status.json`),
+    );
+    await store.recoverIfInterrupted(fence);
+
+    // Codex P1: the deletion census is stale by the time the lock is held.
+    // When the live generation no longer matches the shared base (daemon
+    // indexed, added, or removed members), DEFER the removal — the local
+    // work survives and a later sync re-evaluates with a fresh base.
+    // No shared base (fresh catch-up) = no divergence semantics to protect.
+    const baseStates = baseStatesForGeneration(input.baseStates, shardDirRel);
+    if (baseStates.size > 0) {
+      const localCensus = await censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel);
+      const baseArr = [...baseStates].map(([relPath, state]) => ({ path: relPath, sha256: state.sha256 }));
+      const censusArr = [...localCensus].map(([relPath, sha256]) => ({ path: relPath, sha256 }));
+      const changedSinceBase =
+        censusArr.some((member) => {
+          // A MODIFIED member must also defer: removing the generation would
+          // erase the daemon's rewrite, not just the remote's stale copy.
+          const baseSha = baseStates.get(member.path)?.sha256;
+          return baseSha === undefined || baseSha !== member.sha256;
+        }) || baseArr.some((member) => !localCensus.has(member.path));
+      if (changedSinceBase) {
+        const removedPaths = new Set<string>([...localCensus.keys()]);
+        log.warn(
+          `embedding generation removal ${shardDirRel} deferred: disk diverged from the shared base under the mutation lock; local generation preserved`,
+        );
+        return {
+          shardDirRel,
+          upserted: 0,
+          deleted: 0,
+          writtenStates: new Map<string, OfflineSyncFileState>(),
+          removedPaths,
+          handledPaths: new Set(removedPaths),
+          deferredLocalDivergence: true,
+        };
+      }
+    }
+    const localShardRels = (await readdir(shardDirAbs).catch((err) => {
+      if ((err as NodeJS.ErrnoException).code === "ENOENT") return [] as string[];
+      throw err;
+    }))
+      .filter((name) => EMBEDDING_SHARD_FILE_PATTERN.test(name))
+      .map((name) => `${shardDirRel}/${name}`)
+      .sort();
+    await store.removePublishedGeneration(fence);
+    const removedPaths = new Set(localShardRels);
+    let deleted = removedPaths.size;
+    const markerPresent = await pathExists(markerAbs);
+    if (markerPresent) {
+      removedPaths.add(markerRel);
+      deleted += 1;
+    }
+    // Preserve the replicated tombstones through the configured hooks AFTER
+    // the atomic publication: raw removal must not lose the deletion
+    // revisions, or a downstream sync without base evidence retains stale
+    // entries. EVERY member (shards and marker) is recorded exactly ONCE with
+    // the remote revision when known, so a first hook call without a revision
+    // is never overwritten by an older replicated one. The storage hook
+    // tolerates already-absent paths.
+    let firstError: unknown = null;
+    for (const relPath of removedPaths) {
+      const memberAbs = relPath === markerRel ? markerAbs : await resolveSafeArchiveTarget(root, relPath);
+      const mtimeMs = input.deletionMtimeByPath?.get(relPath) ?? input.now;
+      if (io.deleteFile) {
+        try {
+          await io.deleteFile({
+            root: root.abs,
+            path: relPath,
+            filePath: await resolveSafeArchiveTarget(root, relPath),
+            mtimeMs,
+          });
+        } catch (err) {
+          firstError ??= err;
+          log.warn(`embedding generation removal: tombstone recording failed for ${relPath}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      } else if (relPath === markerRel && markerPresent) {
+        await rm(markerAbs, { force: true });
+      }
+    }
+    if (firstError) throw firstError;
+    return {
+      shardDirRel,
+      upserted: 0,
+      deleted,
+      writtenStates: new Map(),
+      removedPaths,
+      handledPaths: new Set(removedPaths),
+    };
+  });
+}
+
+/** Defer a whole generation when the shared remote base would erase a local change. */
+export function divergedEmbeddingGenerationDeferrals(options: {
+  incomingFiles: readonly { path: string; sha256: string }[];
+  baseFiles: readonly { path: string; sha256: string }[];
+  currentFiles: readonly { path: string; sha256: string }[];
+}): string[] {
+  const base = new Map(options.baseFiles.map((f) => [f.path, f.sha256]));
+  const current = new Map(options.currentFiles.map((f) => [f.path, f.sha256]));
+  const incomingByPath = new Map(options.incomingFiles.map((f) => [f.path, f.sha256]));
+  const sharedDirs = new Set(options.baseFiles.map((f) => embeddingGenerationMembership(f.path)?.shardDir));
+  const conflictedDirs = new Set<string>();
+  for (const relPath of new Set([...base.keys(), ...current.keys(), ...incomingByPath.keys()])) {
+    const membership = embeddingGenerationMembership(relPath);
+    // Without a shared generation base this is an initial remote-authoritative
+    // import, not evidence of local indexing after a synchronized snapshot.
+    if (!membership || !sharedDirs.has(membership.shardDir)) continue;
+    const currentSha = current.get(relPath);
+    if (currentSha !== base.get(relPath) && currentSha !== incomingByPath.get(relPath)) {
+      conflictedDirs.add(membership.shardDir);
+    }
+  }
+  if (conflictedDirs.size === 0) return [];
+  const deferred: string[] = [];
+  // Include local/base members too: an all-absent remote generation still
+  // needs a concrete deferral so its removal cannot erase a local change.
+  for (const relPath of new Set([...base.keys(), ...current.keys(), ...incomingByPath.keys()])) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (membership && conflictedDirs.has(membership.shardDir)) deferred.push(relPath);
+  }
+  return deferred.sort((left, right) => left.localeCompare(right));
+}
+
+// ---------------------------------------------------------------------------
+// Snapshot-side generation routing, extracted from applyOfflineSyncSnapshot
+// for issue #3150 so the changeset path could share the machinery without
+// growing the offline-sync.ts ratchet ceiling. Behavior-preserving.
+// ---------------------------------------------------------------------------
+
+export interface IncomingEmbeddingGenerationApply {
+  transactionResults: EmbeddingGenerationTransactionResult[];
+  deferredGenerationDirs: Set<string>;
+}
+
+export interface IncomingEmbeddingGenerationApplyOptions {
+  root: SafeArchiveRoot;
+  incomingMap: ReadonlyMap<string, OfflineSyncFileState>;
+  incomingBuffers: ReadonlyMap<string, Buffer>;
+  baseMap: ReadonlyMap<string, OfflineSyncFileState>;
+  currentMap: ReadonlyMap<string, OfflineSyncFileState>;
+  deferredPaths: ReadonlySet<string>;
+  omittedGenerationDirs: ReadonlySet<string>;
+  deletionMtimeByPath?: ReadonlyMap<string, number>;
+  io: EmbeddingGenerationTransactionIo;
+  /** True when the caller supplied ANY custom storage IO. */
+  customIoPresent: boolean;
+  now: number;
+}
+
+/**
+ * Snapshot-side generation routing, extracted from applyOfflineSyncSnapshot
+ * for issue #3150 (the changeset path needed the same machinery and the
+ * offline-sync.ts line ratchet had no headroom). Incoming generations replace
+ * as ONE unit through the shared locked transaction. Behavior-preserving —
+ * the #3148 regression suites own this contract.
+ */
+export async function applyIncomingEmbeddingGenerations(
+  options: IncomingEmbeddingGenerationApplyOptions,
+): Promise<IncomingEmbeddingGenerationApply> {
+  const incomingGenerations = detectIncomingEmbeddingGenerations(options.incomingMap.keys());
+  const generationDirs = new Set<string>([
+    ...incomingGenerations.shardDirs,
+    ...incomingGenerations.legacyMarkerDirs,
+  ]);
+  const deferredGenerationDirs = new Set<string>();
+  for (const deferredPath of options.deferredPaths) {
+    const membership = embeddingGenerationMembership(deferredPath);
+    if (membership) deferredGenerationDirs.add(membership.shardDir);
+  }
+  const transactionResults: EmbeddingGenerationTransactionResult[] = [];
+  // Included and omitted cannot describe the same generation.
+  for (const shardDir of generationDirs) {
+    if (options.omittedGenerationDirs.has(shardDir)) {
+      throw new EmbeddingIndexStorageError(
+        `offline sync snapshot both includes and omits embedding generation ${shardDir}; refusing to apply`,
+      );
+    }
+  }
+  // Fail closed BEFORE any publication: custom storage IO (encrypted or
+  // otherwise) without the staging pair would make the generation transaction
+  // fall back to unencrypted raw writes — a plaintext downgrade of the staged
+  // generation. Callers must wire both staging hooks (the CLI does).
+  if (
+    generationDirs.size > 0 &&
+    options.customIoPresent &&
+    !(options.io.writeStagingFile && options.io.readStagingFile)
+  ) {
+    throw new EmbeddingIndexStorageError(
+      "refusing to replace embedding generation: custom storage IO requires both writeStagingFile and readStagingFile hooks; unencrypted raw staging would plaintext-downgrade the index",
+    );
+  }
+  for (const shardDir of generationDirs) {
+    if (deferredGenerationDirs.has(shardDir)) continue;
+    if (
+      incomingGenerations.legacyMarkerDirs.has(shardDir) &&
+      !incomingGenerations.shardDirs.has(shardDir)
+    ) {
+      // Cross-layout: an incoming legacy marker replaces a local sharded
+      // generation as a whole (codex round 4). When shards are ALSO
+      // incoming, the sharded set is the generation and the marker is
+      // written as an inert artifact by the per-file loop below.
+      const markerRel = [...options.incomingMap.keys()].find(
+        (relPath) => embeddingMarkerDirOf(relPath) === shardDir.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length),
+      );
+      if (!markerRel) continue;
+      transactionResults.push(await applyEmbeddingGenerationTransaction({
+        root: options.root,
+        shardDirRel: shardDir,
+        incomingShardPaths: [],
+        incomingMarker: {
+          path: markerRel,
+          sha256: options.incomingMap.get(markerRel)!.sha256,
+          buffer: options.incomingBuffers.get(markerRel) ?? null,
+        },
+        incomingMarkerPresent: false,
+        incomingShardStates: new Map(),
+        incomingBuffers: options.incomingBuffers,
+        baseStates: options.baseMap,
+        io: options.io,
+        now: options.now,
+      }));
+      continue;
+    }
+    transactionResults.push(await applyEmbeddingGenerationTransaction({
+      root: options.root,
+      shardDirRel: shardDir,
+      incomingShardPaths: [...options.incomingMap.keys()]
+        .filter((relPath) => embeddingShardDirOf(relPath) === shardDir)
+        .sort(),
+      incomingMarker: null,
+      incomingMarkerPresent: Boolean(
+        incomingGenerations.legacyMarkerDirs.has(shardDir),
+      ),
+      incomingShardStates: new Map(
+        [...options.incomingMap.values()]
+          .filter((state) => embeddingShardDirOf(state.path) === shardDir)
+          .map((state) => [state.path, state]),
+      ),
+      incomingBuffers: options.incomingBuffers,
+      baseStates: options.baseMap,
+      io: options.io,
+      now: options.now,
+    }));
+  }
+  // Whole-generation removal (codex round 6 P1): when the remote deleted a
+  // previously shared sharded index, the deletion revisions cover EVERY
+  // local member. Remove that generation as ONE atomic swap instead of
+  // deleting shards one-by-one, so a crash or delete failure can never
+  // expose a partial same-identity generation serving missing memories.
+  const localGenerationDirs = new Set<string>();
+  for (const relPath of options.currentMap.keys()) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (membership?.kind === "shard") localGenerationDirs.add(membership.shardDir);
+  }
+  for (const shardDir of localGenerationDirs) {
+    if (
+      generationDirs.has(shardDir) ||
+      deferredGenerationDirs.has(shardDir) ||
+      options.omittedGenerationDirs.has(shardDir)
+    ) {
+      continue;
+    }
+    const localMembers = [...options.currentMap.keys()].filter(
+      (relPath) => embeddingGenerationMembership(relPath)?.shardDir === shardDir,
+    );
+    if (localMembers.length === 0) continue;
+    // Tombstone or shared-base absence authorizes whole-generation removal.
+    const baseMembers = localMembers.filter((relPath) => options.baseMap.has(relPath));
+    // Tombstone evidence is satisfied by the members the REMOTE knew: the
+    // remote cannot tombstone local-only extras or an inert legacy marker,
+    // and the removal sweeps those once the remote deletion is established.
+    // Non-empty same-generation tombstones establish remote absence on
+    // their own: they may name shards already absent locally (removed in an
+    // earlier round), so the set is NOT required to intersect localMembers
+    // (codex round 6, XiOb).
+    const tombstonedMembers =
+      options.deletionMtimeByPath === undefined
+        ? []
+        : [...options.deletionMtimeByPath.keys()].filter(
+            (relPath) => embeddingGenerationMembership(relPath)?.shardDir === shardDir,
+          );
+    const tombstoneCovered = tombstonedMembers.length > 0;
+    const baseEvidenced =
+      baseMembers.length > 0 && baseMembers.every((relPath) => !options.incomingMap.has(relPath));
+    if (!tombstoneCovered && !baseEvidenced) continue;
+    transactionResults.push(await applyEmbeddingGenerationRemoval({
+      root: options.root,
+      shardDirRel: shardDir,
+      io: { deleteFile: options.io.deleteFile, readFile: options.io.readFile },
+      deletionMtimeByPath: options.deletionMtimeByPath,
+      baseStates: options.baseMap,
+      now: options.now,
+    }));
+  }
+  return { transactionResults, deferredGenerationDirs };
+}