@remnic/core 9.69.87 → 9.69.89

package/dist/chunk-AXGJMDBS.js

@@ -0,0 +1,684 @@
+import {
+  EMBEDDING_SHARD_FILE_PATTERN
+} from "./chunk-TYT4TEQV.js";
+import {
+  matchesOfflineSyncDefaultExclude
+} from "./chunk-4NSLQ7EU.js";
+import {
+  resolveSafeArchiveTarget
+} from "./chunk-GCGJW34D.js";
+import {
+  assertEmbeddingGenerationLockHeld,
+  embeddingGenerationLockPath,
+  withEmbeddingGenerationLock
+} from "./chunk-75PT4N7Q.js";
+import {
+  EmbeddingIndexFileStore,
+  EmbeddingIndexStorageError,
+  parseEmbeddingIndexDocument,
+  serializeEmbeddingShard,
+  shardEntriesForIndex,
+  validateShardMembership
+} from "./chunk-UCPGOUDZ.js";
+import {
+  log
+} from "./chunk-3WCASNQZ.js";
+
+// src/offline-sync-embedding-generation.ts
+import { createHash, randomBytes } from "crypto";
+import { lstat, mkdir, readFile, readdir, rm, writeFile } from "fs/promises";
+import path from "path";
+var EMBEDDING_MARKER_BASENAME = "embeddings.json";
+var EMBEDDING_SHARD_DIR_BASENAME = "embeddings";
+function canonicalStateDirPrefix(parts) {
+  if (parts[0] === "state") return "state";
+  if (parts[0] === "namespaces" && parts.length >= 4 && parts[2] === "state") {
+    return `namespaces/${parts[1]}/state`;
+  }
+  return null;
+}
+function embeddingShardDirOf(relPath) {
+  const parts = relPath.split("/");
+  if (parts.length < 3) return null;
+  if (!EMBEDDING_SHARD_FILE_PATTERN.test(parts[parts.length - 1] ?? "")) return null;
+  if (parts[parts.length - 2] !== EMBEDDING_SHARD_DIR_BASENAME) return null;
+  if (canonicalStateDirPrefix(parts) === null) return null;
+  return parts.slice(0, -1).join("/");
+}
+function isEmbeddingGenerationDirPath(dirPath) {
+  const membership = embeddingGenerationMembership(`${dirPath}/shard-0000.json`);
+  return membership !== null && membership.shardDir === dirPath;
+}
+function embeddingMarkerDirOf(relPath) {
+  const parts = relPath.split("/");
+  if ((parts[parts.length - 1] ?? "") !== EMBEDDING_MARKER_BASENAME) return null;
+  if (canonicalStateDirPrefix(parts) === null) return null;
+  return parts.slice(0, -1).join("/");
+}
+function embeddingGenerationMembership(relPath) {
+  const shardDir = embeddingShardDirOf(relPath);
+  if (shardDir) return { kind: "shard", shardDir };
+  const markerDir = embeddingMarkerDirOf(relPath);
+  if (markerDir) return { kind: "marker", shardDir: `${markerDir}/${EMBEDDING_SHARD_DIR_BASENAME}` };
+  return null;
+}
+function detectIncomingEmbeddingGenerations(incomingPaths) {
+  const shardDirs = /* @__PURE__ */ new Set();
+  const legacyMarkerDirs = /* @__PURE__ */ new Set();
+  for (const relPath of incomingPaths) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (!membership) continue;
+    if (membership.kind === "shard") shardDirs.add(membership.shardDir);
+    else legacyMarkerDirs.add(membership.shardDir);
+  }
+  return { shardDirs, legacyMarkerDirs };
+}
+function shardRelPath(shardDirRel, shardIndex) {
+  return `${shardDirRel}/shard-${String(shardIndex).padStart(4, "0")}.json`;
+}
+function describeIndexReadFailure(read) {
+  if (read.outcome === "unreadable") return read.reason;
+  if (read.outcome === "foreign") return "unrecognized index format";
+  return "generation document is absent";
+}
+function sha256Hex(buffer) {
+  return createHash("sha256").update(buffer).digest("hex");
+}
+async function pathExists(absPath) {
+  try {
+    await lstat(absPath);
+    return true;
+  } catch (err) {
+    if (err.code === "ENOENT") return false;
+    throw err;
+  }
+}
+async function hydratedIncomingBuffer(io, root, relPath, expectedSha256, incoming) {
+  if (incoming) return incoming;
+  const filePath = await resolveSafeArchiveTarget(root, relPath);
+  const local = io.readFile ? await io.readFile({ root: root.abs, path: relPath, filePath }) : await readFile(filePath);
+  if (sha256Hex(local) !== expectedSha256) {
+    throw new EmbeddingIndexStorageError(
+      `cannot hydrate unchanged embedding index file ${relPath}: local digest does not match the incoming record; local generation preserved`
+    );
+  }
+  return local;
+}
+function baseStatesForGeneration(base, shardDirRel) {
+  return new Map([...base ?? []].filter(([relPath]) => embeddingGenerationMembership(relPath)?.shardDir === shardDirRel));
+}
+async function censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel) {
+  const census = /* @__PURE__ */ new Map();
+  const readSha = async (relPath, filePath) => sha256Hex(io.readFile ? await io.readFile({ root: root.abs, path: relPath, filePath }) : await readFile(filePath));
+  if (await pathExists(markerAbs)) census.set(markerRel, await readSha(markerRel, markerAbs));
+  const names = await readdir(shardDirAbs).catch((err) => {
+    if (err.code === "ENOENT") return [];
+    throw err;
+  });
+  for (const name of names.filter((candidate) => EMBEDDING_SHARD_FILE_PATTERN.test(candidate)).sort()) {
+    const relPath = `${shardDirRel}/${name}`;
+    census.set(relPath, await readSha(relPath, path.join(shardDirAbs, name)));
+  }
+  return census;
+}
+function incomingGenerationStates(input) {
+  const states = input.incomingShardPaths.map((relPath) => ({
+    path: relPath,
+    sha256: input.incomingShardStates.get(relPath)?.sha256 ?? sha256Hex(input.incomingBuffers.get(relPath) ?? Buffer.alloc(0))
+  }));
+  if (input.incomingMarker) {
+    states.push({ path: input.incomingMarker.path, sha256: input.incomingMarker.sha256 });
+  }
+  return states;
+}
+async function applyEmbeddingGenerationTransaction(input) {
+  const { root, shardDirRel, io } = input;
+  const stateDirRel = shardDirRel.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length);
+  const stateDirAbs = await resolveSafeArchiveTarget(root, stateDirRel);
+  return await withEmbeddingGenerationLock(stateDirAbs, async (section) => {
+    const lockPath = embeddingGenerationLockPath(stateDirAbs);
+    const fence = () => assertEmbeddingGenerationLockHeld(lockPath, section);
+    const markerRel = `${stateDirRel}/${EMBEDDING_MARKER_BASENAME}`;
+    const shardDirAbs = await resolveSafeArchiveTarget(root, shardDirRel);
+    const markerAbs = await resolveSafeArchiveTarget(root, markerRel);
+    const store = new EmbeddingIndexFileStore(
+      markerAbs,
+      shardDirAbs,
+      await resolveSafeArchiveTarget(root, `${stateDirRel}/embedding-fallback-status.json`)
+    );
+    const backupRel = `${stateDirRel}/embeddings.pre-replace.tmp`;
+    await resolveSafeArchiveTarget(root, backupRel);
+    if (await store.recoverIfInterrupted(fence)) {
+      await resolveSafeArchiveTarget(root, shardDirRel);
+    }
+    const justifiedRemovals = input.justifiedRemovals ?? /* @__PURE__ */ new Set();
+    const baseStates = baseStatesForGeneration(input.baseStates, shardDirRel);
+    if (baseStates.size > 0) {
+      const localCensus = await censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel);
+      const baseArr = [...baseStates].map(([relPath, state]) => ({ path: relPath, sha256: state.sha256 }));
+      const censusArr = [...localCensus].map(([relPath, sha256]) => ({ path: relPath, sha256 }));
+      const deferrals = divergedEmbeddingGenerationDeferrals({
+        incomingFiles: incomingGenerationStates(input),
+        // A pre-authorized removal keeps its base entry only while still on
+        // disk (digest auth): present members are compared normally — a
+        // daemon rewrite still defers — while an absent member is already
+        // applied and must not diverge the retry (#3150 review).
+        baseFiles: baseArr.filter((member) => !justifiedRemovals.has(member.path) || localCensus.has(member.path)),
+        currentFiles: censusArr
+      });
+      const incomingShardPaths = new Set(input.incomingShardPaths);
+      const addedOrRemovedSinceBase = censusArr.some((member) => !baseStates.has(member.path) && !justifiedRemovals.has(member.path) && // An incoming member found in the census is the apply's own
+      // already-published work (response-loss retry), not a local
+      // addition; its bytes were verified above.
+      !incomingShardPaths.has(member.path)) || baseArr.some((member) => !localCensus.has(member.path) && !justifiedRemovals.has(member.path));
+      if (deferrals.length > 0 || addedOrRemovedSinceBase) {
+        const handled = new Set(input.incomingShardPaths);
+        if (input.incomingMarker) handled.add(input.incomingMarker.path);
+        if (input.incomingMarkerPresent) handled.add(markerRel);
+        for (const relPath of localCensus.keys()) handled.add(relPath);
+        log.warn(
+          `embedding generation ${shardDirRel} deferred: disk diverged from the shared base under the mutation lock (${deferrals.length > 0 ? deferrals.join(", ") : "generation membership changed since base"}); nothing published, nothing removed`
+        );
+        return {
+          shardDirRel,
+          upserted: 0,
+          deleted: 0,
+          writtenStates: /* @__PURE__ */ new Map(),
+          removedPaths: /* @__PURE__ */ new Set(),
+          handledPaths: handled,
+          deferredLocalDivergence: true
+        };
+      }
+    } else if (input.freshCreateClaim) {
+      const localCensus = await censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel);
+      if (localCensus.size > 0) {
+        const incomingArr = incomingGenerationStates(input).sort((left, right) => left.path.localeCompare(right.path));
+        const censusArr = [...localCensus].map(([relPath, sha256]) => ({ path: relPath, sha256 })).sort((left, right) => left.path.localeCompare(right.path));
+        const identical = censusArr.length === incomingArr.length && censusArr.every((member, index) => member.path === incomingArr[index]?.path && member.sha256 === incomingArr[index]?.sha256);
+        if (!identical) {
+          const handled = new Set(input.incomingShardPaths);
+          if (input.incomingMarker) handled.add(input.incomingMarker.path);
+          if (input.incomingMarkerPresent) handled.add(markerRel);
+          for (const relPath of localCensus.keys()) handled.add(relPath);
+          log.warn(
+            `embedding generation ${shardDirRel} deferred: base-less creation does not match the existing local generation under the mutation lock; nothing published, nothing removed`
+          );
+          return {
+            shardDirRel,
+            upserted: 0,
+            deleted: 0,
+            writtenStates: /* @__PURE__ */ new Map(),
+            removedPaths: /* @__PURE__ */ new Set(),
+            handledPaths: handled,
+            deferredLocalDivergence: true
+          };
+        }
+      }
+    }
+    const published = /* @__PURE__ */ new Map();
+    let identity = null;
+    if (input.incomingMarker) {
+      const markerBuffer = await hydratedIncomingBuffer(
+        io,
+        root,
+        input.incomingMarker.path,
+        input.incomingMarker.sha256,
+        input.incomingMarker.buffer
+      );
+      const read = parseEmbeddingIndexDocument(markerBuffer.toString("utf-8"));
+      if (read.outcome !== "ok") {
+        throw new EmbeddingIndexStorageError(
+          `refusing incoming embedding index generation ${markerRel}: ${describeIndexReadFailure(read)}; local generation preserved`
+        );
+      }
+      identity = { provider: read.file.provider, model: read.file.model };
+      const groups = shardEntriesForIndex(read.file);
+      for (const [shardIndex, entries] of groups) {
+        if (Object.keys(entries).length === 0) continue;
+        published.set(
+          shardRelPath(shardDirRel, shardIndex),
+          Buffer.from(serializeEmbeddingShard(read.file, shardIndex, entries), "utf-8")
+        );
+      }
+    } else {
+      for (const relPath of input.incomingShardPaths) {
+        const state = input.incomingShardStates.get(relPath);
+        const buffer = await hydratedIncomingBuffer(
+          io,
+          root,
+          relPath,
+          state?.sha256 ?? "",
+          input.incomingBuffers.get(relPath)
+        );
+        const read = parseEmbeddingIndexDocument(buffer.toString("utf-8"));
+        if (read.outcome !== "ok") {
+          throw new EmbeddingIndexStorageError(
+            `refusing incoming embedding index shard ${relPath}: ${describeIndexReadFailure(read)}; local generation preserved`
+          );
+        }
+        validateShardMembership(relPath.split("/").pop() ?? "", read.file);
+        const shardIdentity = { provider: read.file.provider, model: read.file.model };
+        if (!identity) identity = shardIdentity;
+        if (identity.provider !== shardIdentity.provider || identity.model !== shardIdentity.model) {
+          throw new EmbeddingIndexStorageError(
+            `refusing mixed-identity incoming embedding index generation in ${shardDirRel}: ${relPath} is ${shardIdentity.provider}/${shardIdentity.model} but the generation is ${identity.provider}/${identity.model}; local generation preserved`
+          );
+        }
+        published.set(relPath, buffer);
+      }
+      if (published.size === 0) {
+        throw new EmbeddingIndexStorageError(
+          `incoming embedding index generation ${shardDirRel} is empty; local generation preserved`
+        );
+      }
+    }
+    const stagingRel = `${stateDirRel}/embeddings.staging.tmp-sync-${process.pid}-${input.now}-${randomBytes(8).toString("hex")}`;
+    const stagingAbs = await resolveSafeArchiveTarget(root, stagingRel);
+    try {
+      await mkdir(stagingAbs, { recursive: true });
+      for (const [relPath, buffer] of published) {
+        const stagedRel = `${stagingRel}/${path.basename(relPath)}`;
+        const stagedAbs = await resolveSafeArchiveTarget(root, stagedRel);
+        const finalAbs = await resolveSafeArchiveTarget(root, relPath);
+        if (io.writeStagingFile) {
+          await io.writeStagingFile({
+            root: root.abs,
+            path: stagedRel,
+            filePath: stagedAbs,
+            content: buffer,
+            finalAadFilePath: finalAbs
+          });
+        } else {
+          await writeFile(stagedAbs, buffer, { mode: 384 });
+        }
+      }
+      for (const [relPath, buffer] of published) {
+        const stagedRel = `${stagingRel}/${path.basename(relPath)}`;
+        const stagedAbs = await resolveSafeArchiveTarget(root, stagedRel);
+        const staged = io.readStagingFile ? await io.readStagingFile({
+          root: root.abs,
+          path: stagedRel,
+          filePath: stagedAbs,
+          finalAadFilePath: await resolveSafeArchiveTarget(root, relPath)
+        }) : await readFile(stagedAbs);
+        if (!staged.equals(buffer)) {
+          throw new EmbeddingIndexStorageError(
+            `staged embedding index file ${stagedRel} does not match the incoming content; local generation preserved`
+          );
+        }
+      }
+      const localShardRels = (await readdir(shardDirAbs).catch((err) => {
+        if (err.code === "ENOENT") return [];
+        throw err;
+      })).filter((name) => EMBEDDING_SHARD_FILE_PATTERN.test(name)).map((name) => `${shardDirRel}/${name}`).sort();
+      await store.publishSwappedGeneration(stagingAbs, fence);
+      const writtenStates = /* @__PURE__ */ new Map();
+      for (const [relPath, buffer] of published) {
+        writtenStates.set(relPath, {
+          path: relPath,
+          sha256: sha256Hex(buffer),
+          bytes: buffer.length,
+          mtimeMs: input.now
+        });
+      }
+      const removedPaths = new Set(localShardRels.filter((rel) => !published.has(rel)));
+      let deleted = removedPaths.size;
+      let markerCleanupFailed = false;
+      if (await pathExists(markerAbs) && !input.incomingMarker && !input.incomingMarkerPresent) {
+        try {
+          if (io.deleteFile) await io.deleteFile({ root: root.abs, path: markerRel, filePath: markerAbs });
+          else await rm(markerAbs, { force: true });
+          removedPaths.add(markerRel);
+          deleted += 1;
+        } catch (err) {
+          markerCleanupFailed = true;
+          log.warn(`embedding generation ${shardDirRel}: marker cleanup deferred after publish: ${err}`);
+        }
+      }
+      if (input.deletionMtimeByPath !== void 0 && io.deleteFile) {
+        let firstTombstoneError = null;
+        for (const relPath of removedPaths) {
+          try {
+            await io.deleteFile({
+              root: root.abs,
+              path: relPath,
+              filePath: await resolveSafeArchiveTarget(root, relPath),
+              mtimeMs: input.deletionMtimeByPath.get(relPath) ?? input.now
+            });
+          } catch (err) {
+            firstTombstoneError ??= err;
+            log.warn(`embedding generation ${shardDirRel}: tombstone recording failed for ${relPath}: ${err instanceof Error ? err.message : String(err)}`);
+          }
+        }
+        if (firstTombstoneError) throw firstTombstoneError;
+      }
+      return {
+        shardDirRel,
+        upserted: published.size,
+        deleted,
+        writtenStates,
+        removedPaths,
+        handledPaths: /* @__PURE__ */ new Set([...published.keys(), ...removedPaths, ...markerCleanupFailed ? [markerRel] : []])
+      };
+    } catch (err) {
+      await rm(stagingAbs, { recursive: true, force: true }).catch(() => void 0);
+      throw err;
+    }
+  });
+}
+async function computeOmittedEmbeddingGenerationPaths(options) {
+  const omittedPaths = /* @__PURE__ */ new Set();
+  const omittedDirs = [];
+  const stateDirs = ["state"];
+  let namespaceNames = [];
+  try {
+    namespaceNames = (await readdir(path.join(options.rootAbs, "namespaces"), { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+  } catch {
+  }
+  for (const ns of namespaceNames) stateDirs.push(`namespaces/${ns}/state`);
+  const isExcluded = options.isExcludedRelPath ?? ((relPath) => matchesOfflineSyncDefaultExclude(relPath) || Boolean(options.userExcludeRegexps?.some((re) => re.test(relPath))));
+  for (const stateDir of stateDirs) {
+    const markerRel = `${stateDir}/${EMBEDDING_MARKER_BASENAME}`;
+    const shardDirRel = `${stateDir}/${EMBEDDING_SHARD_DIR_BASENAME}`;
+    const shardDirAbs = path.join(options.rootAbs, ...shardDirRel.split("/"));
+    const memberRels = [];
+    if (await pathExists(path.join(options.rootAbs, ...markerRel.split("/")))) {
+      memberRels.push(markerRel);
+    }
+    let shardNames = [];
+    try {
+      shardNames = (await readdir(shardDirAbs, { withFileTypes: true })).filter((entry) => entry.isFile() && EMBEDDING_SHARD_FILE_PATTERN.test(entry.name)).map((entry) => entry.name);
+    } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+    }
+    for (const name of shardNames) memberRels.push(`${shardDirRel}/${name}`);
+    for (const tombstoned of options.tombstonedPaths ?? []) {
+      const membership = embeddingGenerationMembership(tombstoned);
+      if (membership?.shardDir === shardDirRel && !memberRels.includes(tombstoned)) {
+        memberRels.push(tombstoned);
+      }
+    }
+    if (memberRels.length === 0) continue;
+    const excludedMembers = [];
+    for (const relPath of memberRels) {
+      let excluded = isExcluded(relPath);
+      if (!excluded && options.excludeFile) {
+        excluded = await options.excludeFile({
+          root: options.rootAbs,
+          path: relPath,
+          filePath: path.join(options.rootAbs, ...relPath.split("/"))
+        });
+      }
+      if (excluded) excludedMembers.push(relPath);
+    }
+    if (excludedMembers.length === 0) continue;
+    omittedDirs.push(shardDirRel);
+    omittedPaths.add(shardDirRel);
+    for (const relPath of memberRels) omittedPaths.add(relPath);
+  }
+  return { omittedPaths, omittedDirs };
+}
+async function assertEmbeddingGenerationStillIncluded(rootAbs, shardDirRel, excludeFile, isExcludedRelPath, tombstonedPaths) {
+  const current = await computeOmittedEmbeddingGenerationPaths({ rootAbs, excludeFile, isExcludedRelPath, tombstonedPaths });
+  if (current.omittedDirs.includes(shardDirRel)) {
+    throw new EmbeddingIndexStorageError(
+      `embedding generation ${shardDirRel} became partially excluded during snapshot capture; retry with a fresh omission header`
+    );
+  }
+}
+function isPathInOmittedEmbeddingGeneration(relPath, omittedDirs) {
+  const membership = embeddingGenerationMembership(relPath);
+  return membership !== null && omittedDirs.has(membership.shardDir);
+}
+async function resolvePushEmbeddingGenerationState(options) {
+  const omission = await computeOmittedEmbeddingGenerationPaths({
+    rootAbs: options.rootAbs,
+    userExcludeRegexps: options.userExcludeRegexps,
+    excludeFile: options.excludeFile,
+    // RAW pre-filter tombstones: a deleted member's exclusion must omit the
+    // whole generation even though the member is already absent on disk.
+    tombstonedPaths: options.deletions.map((deletion) => deletion.path),
+    isExcludedRelPath: (relPath) => matchesOfflineSyncDefaultExclude(relPath) || Boolean(options.userExcludeRegexps?.some((re) => re.test(relPath)))
+  });
+  const omittedDirs = new Set(omission.omittedDirs);
+  const deletions = options.deletions.filter((deletion) => {
+    if (matchesOfflineSyncDefaultExclude(deletion.path)) return false;
+    if (options.userExcludeRegexps?.some((re) => re.test(deletion.path))) return false;
+    const membership = embeddingGenerationMembership(deletion.path);
+    if (membership && omittedDirs.has(membership.shardDir)) return false;
+    return true;
+  });
+  return { omittedDirs: omission.omittedDirs, omittedPaths: omission.omittedPaths, deletions };
+}
+function storageBackedIndexStoreIo(storage) {
+  return {
+    readUtf8: async (filePath) => (await storage.readOfflineSyncFile(filePath)).toString("utf-8"),
+    writeUtf8: (filePath, contents, opts) => storage.writeOfflineSyncStagingFile(
+      filePath,
+      Buffer.from(contents, "utf-8"),
+      opts?.finalAadFilePath === void 0 ? void 0 : { aadFilePath: opts.finalAadFilePath }
+    )
+  };
+}
+async function applyEmbeddingGenerationRemoval(input) {
+  const { root, shardDirRel, io } = input;
+  const stateDirRel = shardDirRel.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length);
+  const stateDirAbs = await resolveSafeArchiveTarget(root, stateDirRel);
+  return await withEmbeddingGenerationLock(stateDirAbs, async (section) => {
+    const lockPath = embeddingGenerationLockPath(stateDirAbs);
+    const fence = () => assertEmbeddingGenerationLockHeld(lockPath, section);
+    const markerRel = `${stateDirRel}/${EMBEDDING_MARKER_BASENAME}`;
+    const shardDirAbs = await resolveSafeArchiveTarget(root, shardDirRel);
+    const markerAbs = await resolveSafeArchiveTarget(root, markerRel);
+    const store = new EmbeddingIndexFileStore(
+      markerAbs,
+      shardDirAbs,
+      await resolveSafeArchiveTarget(root, `${stateDirRel}/embedding-fallback-status.json`)
+    );
+    await store.recoverIfInterrupted(fence);
+    const baseStates = baseStatesForGeneration(input.baseStates, shardDirRel);
+    if (baseStates.size > 0) {
+      const localCensus = await censusGenerationState(io, root, markerRel, markerAbs, shardDirAbs, shardDirRel);
+      const baseArr = [...baseStates].map(([relPath, state]) => ({ path: relPath, sha256: state.sha256 }));
+      const censusArr = [...localCensus].map(([relPath, sha256]) => ({ path: relPath, sha256 }));
+      const changedSinceBase = censusArr.some((member) => {
+        const baseSha = baseStates.get(member.path)?.sha256;
+        return baseSha === void 0 || baseSha !== member.sha256;
+      }) || baseArr.some((member) => !localCensus.has(member.path));
+      if (changedSinceBase) {
+        const removedPaths2 = /* @__PURE__ */ new Set([...localCensus.keys()]);
+        log.warn(
+          `embedding generation removal ${shardDirRel} deferred: disk diverged from the shared base under the mutation lock; local generation preserved`
+        );
+        return {
+          shardDirRel,
+          upserted: 0,
+          deleted: 0,
+          writtenStates: /* @__PURE__ */ new Map(),
+          removedPaths: removedPaths2,
+          handledPaths: new Set(removedPaths2),
+          deferredLocalDivergence: true
+        };
+      }
+    }
+    const localShardRels = (await readdir(shardDirAbs).catch((err) => {
+      if (err.code === "ENOENT") return [];
+      throw err;
+    })).filter((name) => EMBEDDING_SHARD_FILE_PATTERN.test(name)).map((name) => `${shardDirRel}/${name}`).sort();
+    await store.removePublishedGeneration(fence);
+    const removedPaths = new Set(localShardRels);
+    let deleted = removedPaths.size;
+    const markerPresent = await pathExists(markerAbs);
+    if (markerPresent) {
+      removedPaths.add(markerRel);
+      deleted += 1;
+    }
+    let firstError = null;
+    for (const relPath of removedPaths) {
+      const memberAbs = relPath === markerRel ? markerAbs : await resolveSafeArchiveTarget(root, relPath);
+      const mtimeMs = input.deletionMtimeByPath?.get(relPath) ?? input.now;
+      if (io.deleteFile) {
+        try {
+          await io.deleteFile({
+            root: root.abs,
+            path: relPath,
+            filePath: await resolveSafeArchiveTarget(root, relPath),
+            mtimeMs
+          });
+        } catch (err) {
+          firstError ??= err;
+          log.warn(`embedding generation removal: tombstone recording failed for ${relPath}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      } else if (relPath === markerRel && markerPresent) {
+        await rm(markerAbs, { force: true });
+      }
+    }
+    if (firstError) throw firstError;
+    return {
+      shardDirRel,
+      upserted: 0,
+      deleted,
+      writtenStates: /* @__PURE__ */ new Map(),
+      removedPaths,
+      handledPaths: new Set(removedPaths)
+    };
+  });
+}
+function divergedEmbeddingGenerationDeferrals(options) {
+  const base = new Map(options.baseFiles.map((f) => [f.path, f.sha256]));
+  const current = new Map(options.currentFiles.map((f) => [f.path, f.sha256]));
+  const incomingByPath = new Map(options.incomingFiles.map((f) => [f.path, f.sha256]));
+  const sharedDirs = new Set(options.baseFiles.map((f) => embeddingGenerationMembership(f.path)?.shardDir));
+  const conflictedDirs = /* @__PURE__ */ new Set();
+  for (const relPath of /* @__PURE__ */ new Set([...base.keys(), ...current.keys(), ...incomingByPath.keys()])) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (!membership || !sharedDirs.has(membership.shardDir)) continue;
+    const currentSha = current.get(relPath);
+    if (currentSha !== base.get(relPath) && currentSha !== incomingByPath.get(relPath)) {
+      conflictedDirs.add(membership.shardDir);
+    }
+  }
+  if (conflictedDirs.size === 0) return [];
+  const deferred = [];
+  for (const relPath of /* @__PURE__ */ new Set([...base.keys(), ...current.keys(), ...incomingByPath.keys()])) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (membership && conflictedDirs.has(membership.shardDir)) deferred.push(relPath);
+  }
+  return deferred.sort((left, right) => left.localeCompare(right));
+}
+async function applyIncomingEmbeddingGenerations(options) {
+  const incomingGenerations = detectIncomingEmbeddingGenerations(options.incomingMap.keys());
+  const generationDirs = /* @__PURE__ */ new Set([
+    ...incomingGenerations.shardDirs,
+    ...incomingGenerations.legacyMarkerDirs
+  ]);
+  const deferredGenerationDirs = /* @__PURE__ */ new Set();
+  for (const deferredPath of options.deferredPaths) {
+    const membership = embeddingGenerationMembership(deferredPath);
+    if (membership) deferredGenerationDirs.add(membership.shardDir);
+  }
+  const transactionResults = [];
+  for (const shardDir of generationDirs) {
+    if (options.omittedGenerationDirs.has(shardDir)) {
+      throw new EmbeddingIndexStorageError(
+        `offline sync snapshot both includes and omits embedding generation ${shardDir}; refusing to apply`
+      );
+    }
+  }
+  if (generationDirs.size > 0 && options.customIoPresent && !(options.io.writeStagingFile && options.io.readStagingFile)) {
+    throw new EmbeddingIndexStorageError(
+      "refusing to replace embedding generation: custom storage IO requires both writeStagingFile and readStagingFile hooks; unencrypted raw staging would plaintext-downgrade the index"
+    );
+  }
+  for (const shardDir of generationDirs) {
+    if (deferredGenerationDirs.has(shardDir)) continue;
+    if (incomingGenerations.legacyMarkerDirs.has(shardDir) && !incomingGenerations.shardDirs.has(shardDir)) {
+      const markerRel = [...options.incomingMap.keys()].find(
+        (relPath) => embeddingMarkerDirOf(relPath) === shardDir.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length)
+      );
+      if (!markerRel) continue;
+      transactionResults.push(await applyEmbeddingGenerationTransaction({
+        root: options.root,
+        shardDirRel: shardDir,
+        incomingShardPaths: [],
+        incomingMarker: {
+          path: markerRel,
+          sha256: options.incomingMap.get(markerRel).sha256,
+          buffer: options.incomingBuffers.get(markerRel) ?? null
+        },
+        incomingMarkerPresent: false,
+        incomingShardStates: /* @__PURE__ */ new Map(),
+        incomingBuffers: options.incomingBuffers,
+        baseStates: options.baseMap,
+        io: options.io,
+        now: options.now
+      }));
+      continue;
+    }
+    transactionResults.push(await applyEmbeddingGenerationTransaction({
+      root: options.root,
+      shardDirRel: shardDir,
+      incomingShardPaths: [...options.incomingMap.keys()].filter((relPath) => embeddingShardDirOf(relPath) === shardDir).sort(),
+      incomingMarker: null,
+      incomingMarkerPresent: Boolean(
+        incomingGenerations.legacyMarkerDirs.has(shardDir)
+      ),
+      incomingShardStates: new Map(
+        [...options.incomingMap.values()].filter((state) => embeddingShardDirOf(state.path) === shardDir).map((state) => [state.path, state])
+      ),
+      incomingBuffers: options.incomingBuffers,
+      baseStates: options.baseMap,
+      io: options.io,
+      now: options.now
+    }));
+  }
+  const localGenerationDirs = /* @__PURE__ */ new Set();
+  for (const relPath of options.currentMap.keys()) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (membership?.kind === "shard") localGenerationDirs.add(membership.shardDir);
+  }
+  for (const shardDir of localGenerationDirs) {
+    if (generationDirs.has(shardDir) || deferredGenerationDirs.has(shardDir) || options.omittedGenerationDirs.has(shardDir)) {
+      continue;
+    }
+    const localMembers = [...options.currentMap.keys()].filter(
+      (relPath) => embeddingGenerationMembership(relPath)?.shardDir === shardDir
+    );
+    if (localMembers.length === 0) continue;
+    const baseMembers = localMembers.filter((relPath) => options.baseMap.has(relPath));
+    const tombstonedMembers = options.deletionMtimeByPath === void 0 ? [] : [...options.deletionMtimeByPath.keys()].filter(
+      (relPath) => embeddingGenerationMembership(relPath)?.shardDir === shardDir
+    );
+    const tombstoneCovered = tombstonedMembers.length > 0;
+    const baseEvidenced = baseMembers.length > 0 && baseMembers.every((relPath) => !options.incomingMap.has(relPath));
+    if (!tombstoneCovered && !baseEvidenced) continue;
+    transactionResults.push(await applyEmbeddingGenerationRemoval({
+      root: options.root,
+      shardDirRel: shardDir,
+      io: { deleteFile: options.io.deleteFile, readFile: options.io.readFile },
+      deletionMtimeByPath: options.deletionMtimeByPath,
+      baseStates: options.baseMap,
+      now: options.now
+    }));
+  }
+  return { transactionResults, deferredGenerationDirs };
+}
+
+export {
+  EMBEDDING_SHARD_DIR_BASENAME,
+  embeddingShardDirOf,
+  isEmbeddingGenerationDirPath,
+  embeddingMarkerDirOf,
+  embeddingGenerationMembership,
+  detectIncomingEmbeddingGenerations,
+  applyEmbeddingGenerationTransaction,
+  computeOmittedEmbeddingGenerationPaths,
+  assertEmbeddingGenerationStillIncluded,
+  isPathInOmittedEmbeddingGeneration,
+  resolvePushEmbeddingGenerationState,
+  storageBackedIndexStoreIo,
+  applyEmbeddingGenerationRemoval,
+  divergedEmbeddingGenerationDeferrals,
+  applyIncomingEmbeddingGenerations
+};
+//# sourceMappingURL=chunk-AXGJMDBS.js.map