@remnic/core 9.69.86 → 9.69.88

package/dist/{chunk-WQZZDN5M.js → chunk-OCFXVDBZ.js}

@@ -2,24 +2,61 @@ import {
   DEFAULT_TRANSFER_EXCLUDE_DIRS
 } from "./chunk-UI3NYK34.js";
 import {
-  prepareSafeArchiveRoot,
-  resolveSafeArchiveTarget,
-  sha256Bytes,
-  validateArchiveRelativePath
-} from "./chunk-GCGJW34D.js";
+  cleanupOfflineUpload,
+  digestOfflineUploadStagingContent,
+  hashText,
+  pruneOfflineUploadStaging,
+  writeOfflineUploadChunk,
+  writeSafeFileFromUpload
+} from "./chunk-ETUZFFPU.js";
+import {
+  EMBEDDING_SHARD_DIR_BASENAME,
+  applyEmbeddingGenerationRemoval,
+  applyEmbeddingGenerationTransaction,
+  assertEmbeddingGenerationStillIncluded,
+  computeOmittedEmbeddingGenerationPaths,
+  detectIncomingEmbeddingGenerations,
+  divergedEmbeddingGenerationDeferrals,
+  embeddingGenerationMembership,
+  embeddingMarkerDirOf,
+  embeddingShardDirOf,
+  isEmbeddingGenerationDirPath,
+  isPathInOmittedEmbeddingGeneration
+} from "./chunk-K6T33R4Q.js";
 import {
-  DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS
-} from "./chunk-AREGMX2J.js";
+  matchesOfflineSyncDefaultExclude
+} from "./chunk-4NSLQ7EU.js";
 import {
+  OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS,
+  SYNC_INTERNAL_DIR,
+  assertOfflineSyncMtimeMs,
   isEncryptedOfflineSyncFile,
   plainFileDigest,
   readPlainOfflineSyncFileChunk,
+  setSafeFileMtime,
   sha256OfflineSyncFile,
   shouldExcludeOfflineSyncFile
-} from "./chunk-3BTFBQLO.js";
+} from "./chunk-EFHNFTWB.js";
+import {
+  prepareSafeArchiveRoot,
+  resolveSafeArchiveTarget,
+  sha256Bytes,
+  validateArchiveRelativePath
+} from "./chunk-GCGJW34D.js";
+import {
+  isCanonicalRuntimeStatePath,
+  shouldDeleteAbsentIncomingOfflineRuntimeFile,
+  shouldPreferIncomingOfflineRuntimeFile
+} from "./chunk-TYT4TEQV.js";
+import {
+  withEmbeddingGenerationLock,
+  withEmbeddingGenerationLockIter
+} from "./chunk-75PT4N7Q.js";
+import {
+  EmbeddingIndexStorageError
+} from "./chunk-UCPGOUDZ.js";
 import {
   CENSUS_MAX_MTIME_MS,
-  isCensusMtimeMs,
   isSha256Hex
 } from "./chunk-LQWOFU6L.js";
 import {
@@ -27,18 +64,15 @@ import {
 } from "./chunk-P7FMDTKL.js";
 
 // src/offline-sync.ts
-import { createHash, randomUUID } from "crypto";
+import { randomUUID } from "crypto";
 import {
   lstat,
   mkdir,
-  open,
   readdir,
   readFile,
   rename,
-  rm,
   stat,
   unlink,
-  utimes,
   writeFile
 } from "fs/promises";
 import path from "path";
@@ -50,9 +84,6 @@ var OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES = 8 * 1024 * 1024;
 var OFFLINE_SYNC_APPLY_MAX_BODY_BYTES = 16 * 1024 * 1024;
 var OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES = 64 * 1024 * 1024;
 var OFFLINE_SYNC_MAX_MTIME_MS = CENSUS_MAX_MTIME_MS;
-var SYNC_INTERNAL_DIR = ".offline-sync";
-var OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1e3;
-var OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS = 1e3;
 var OFFLINE_SYNC_FAST_BASE_CTIME_TOLERANCE_MS = 1;
 var EXCLUDED_FILE_NAMES = /* @__PURE__ */ new Set([
   ".sync-state.json"
@@ -61,45 +92,6 @@ var EXCLUDED_FILE_PREFIXES = [
   ".remnic-sync.",
   ".remnic-sync-state."
 ];
-function globToRegExp(glob) {
-  if (typeof glob !== "string" || glob.length === 0) {
-    throw new Error("offlineSyncExcludes entry must be a non-empty string");
-  }
-  if (glob.includes("\0")) {
-    throw new Error("offlineSyncExcludes entry must not contain NUL bytes");
-  }
-  let source = "";
-  for (let i = 0; i < glob.length; i += 1) {
-    const ch = glob[i];
-    if (ch === "*") {
-      if (glob[i + 1] === "*") {
-        if (glob[i + 2] === "/") {
-          source += "(?:.*/)?";
-          i += 2;
-          continue;
-        }
-        source += ".*";
-        i += 1;
-        continue;
-      }
-      source += "[^/]*";
-      continue;
-    }
-    if (ch === "?") {
-      source += "[^/]";
-      continue;
-    }
-    if (ch === "/") {
-      source += "/";
-      continue;
-    }
-    source += ch.replace(/[\\^$.+()|{}\[\]]/g, "\\$&");
-  }
-  return new RegExp(`^${source}$`);
-}
-function hashText(value) {
-  return createHash("sha256").update(value).digest("hex");
-}
 function sha256Buffer(buffer) {
   return sha256Bytes(buffer);
 }
@@ -122,19 +114,6 @@ function assertNonNegativeInteger(value, field) {
   }
   return value;
 }
-function assertNonNegativeFinite(value, field) {
-  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
-    throw new Error(`${field} must be a non-negative finite number`);
-  }
-  return value;
-}
-function assertOfflineSyncMtimeMs(value, field) {
-  const mtimeMs = assertNonNegativeFinite(value, field);
-  if (!isCensusMtimeMs(mtimeMs)) {
-    throw new Error(`${field} must be within JavaScript Date range`);
-  }
-  return mtimeMs;
-}
 function assertBoolean(value, field) {
   if (typeof value !== "boolean") {
     throw new Error(`${field} must be a boolean`);
@@ -206,12 +185,17 @@ function snapshotBuilderDeletions(options) {
   if (deletions.length === 0) return deletions;
   const presentPaths = new Set(options.files.map((file) => file.path.toLowerCase()));
   const scopedPaths = options.paths ? new Set(options.paths.map((relPath) => relPath.toLowerCase())) : void 0;
-  return deletions.filter((deletion) => (scopedPaths === void 0 || scopedPaths.has(deletion.path.toLowerCase())) && !presentPaths.has(deletion.path.toLowerCase()) && !(options.excludeNodeLocalState === false ? shouldExcludeRelPath(deletion.path, options.includeTranscripts) : shouldExcludePushRelPath(
+  return deletions.filter((deletion) => (scopedPaths === void 0 || scopedPaths.has(deletion.path.toLowerCase())) && !presentPaths.has(deletion.path.toLowerCase()) && !isDeletionInOmittedGeneration(deletion.path, options.omittedGenerationDirs) && !(options.excludeNodeLocalState === false ? shouldExcludeRelPath(deletion.path, options.includeTranscripts) : shouldExcludePushRelPath(
     deletion.path,
     options.includeTranscripts,
     options.userExcludeRegexps
   )));
 }
+function isDeletionInOmittedGeneration(relPath, omittedGenerationDirs) {
+  if (!omittedGenerationDirs || omittedGenerationDirs.size === 0) return false;
+  const membership = embeddingGenerationMembership(relPath);
+  return membership !== null && omittedGenerationDirs.has(membership.shardDir);
+}
 async function filterOfflineSyncDeletionRevisions(options) {
   const deletions = normalizeDeletionRevisions(options.deletions, "deletions");
   if (deletions === void 0) throw new Error("deletions must be an array");
@@ -222,9 +206,16 @@ async function filterOfflineSyncDeletionRevisions(options) {
     "root"
   );
   const includeTranscripts = options.includeTranscripts !== false;
+  const omission = await computeOmittedEmbeddingGenerationPaths({
+    rootAbs: root.abs,
+    userExcludeRegexps: options.userExcludeRegexps,
+    excludeFile: options.excludeFile,
+    isExcludedRelPath: (relPath) => shouldExcludePushRelPath(relPath, includeTranscripts, options.userExcludeRegexps)
+  });
+  const omittedDirs = new Set(omission.omittedDirs);
   const filtered = [];
   for (const deletion of deletions) {
-    if (shouldExcludePushRelPath(deletion.path, includeTranscripts, options.userExcludeRegexps)) {
+    if (shouldExcludePushRelPath(deletion.path, includeTranscripts, options.userExcludeRegexps) || isDeletionInOmittedGeneration(deletion.path, omittedDirs)) {
       continue;
     }
     const filePath = path.join(root.abs, ...deletion.path.split("/"));
@@ -272,6 +263,22 @@ function normalizeOfflineSyncSnapshot(input, options = {}) {
   if (excludedPath) {
     throw new Error(`offline sync snapshot contains excluded path: ${excludedPath}`);
   }
+  let omittedEmbeddingGenerationDirs;
+  if (obj.omittedEmbeddingGenerationDirs !== void 0) {
+    if (!Array.isArray(obj.omittedEmbeddingGenerationDirs)) {
+      throw new Error("offline sync snapshot omittedEmbeddingGenerationDirs must be an array");
+    }
+    const seenDirs = /* @__PURE__ */ new Set();
+    for (const [index, entry] of obj.omittedEmbeddingGenerationDirs.entries()) {
+      if (typeof entry !== "string" || !isEmbeddingGenerationDirPath(entry) || seenDirs.has(entry)) {
+        throw new Error(
+          `omittedEmbeddingGenerationDirs[${index}] must be a unique canonical embeddings generation dir`
+        );
+      }
+      seenDirs.add(entry);
+    }
+    omittedEmbeddingGenerationDirs = [...seenDirs].sort((left, right) => left.localeCompare(right));
+  }
   return {
     format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
     schemaVersion: 1,
@@ -279,7 +286,8 @@ function normalizeOfflineSyncSnapshot(input, options = {}) {
     sourceId,
     includeTranscripts,
     files,
-    ...deletions === void 0 ? {} : { deletions }
+    ...deletions === void 0 ? {} : { deletions },
+    ...omittedEmbeddingGenerationDirs === void 0 ? {} : { omittedEmbeddingGenerationDirs }
   };
 }
 function normalizeOfflineSyncChangeset(input) {
@@ -404,79 +412,12 @@ function shouldExcludePushRelPath(relPosix, includeTranscripts, userExcludeRegex
   if (userExcludeRegexps && userExcludeRegexps.some((re) => re.test(relPosix))) return true;
   return false;
 }
-var DEFAULT_OFFLINE_SYNC_EXCLUDE_REGEXPS = DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS.map((glob) => globToRegExp(glob));
-function matchesOfflineSyncDefaultExclude(relPosix) {
-  for (const regexp of DEFAULT_OFFLINE_SYNC_EXCLUDE_REGEXPS) {
-    if (regexp.test(relPosix)) return true;
-  }
-  return false;
-}
-function compileOfflineSyncExcludeGlobs(globs) {
-  const out = [];
-  for (const entry of globs) {
-    if (typeof entry !== "string" || entry.length === 0) {
-      throw new Error("offlineSyncExcludes must contain only non-empty strings");
-    }
-    out.push(globToRegExp(entry));
-  }
-  return out;
-}
-function parseOfflineSyncExcludes(raw) {
-  if (raw === void 0 || raw === null) return [];
-  if (!Array.isArray(raw)) {
-    throw new Error(
-      `offlineSyncExcludes must be an array of non-empty glob strings; got ${typeof raw}`
-    );
-  }
-  for (const entry of raw) {
-    if (typeof entry !== "string" || entry.trim().length === 0) {
-      throw new Error(
-        "offlineSyncExcludes must contain only non-empty glob strings"
-      );
-    }
-  }
-  const globs = raw.map((entry) => entry.trim());
-  compileOfflineSyncExcludeGlobs(globs);
-  return globs;
-}
 function shouldIgnoreIncomingRuntimePath(relPosix) {
   if (isInternalRemnicStatePath(relPosix)) return true;
   const parts = relPosix.split("/");
   const basename = parts[parts.length - 1] ?? "";
   return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
 }
-function isCanonicalRuntimeStatePath(parts) {
-  if (parts[0] === "state") return true;
-  return parts[0] === "namespaces" && parts.length >= 4 && parts[2] === "state";
-}
-var REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES = /* @__PURE__ */ new Set([
-  ".artifact-write-version.log",
-  ".memory-status-version.log",
-  "buffer.json",
-  "embeddings.json",
-  "index_time.json",
-  "last_intent.json",
-  "last_qmd_recall.json",
-  "last_recall.json",
-  "lcm.sqlite-shm",
-  "lcm.sqlite-wal",
-  "memory-lifecycle-ledger.jsonl",
-  "recall_impressions.jsonl"
-]);
-var ABSENT_INCOMING_RUNTIME_DELETE_FILES = /* @__PURE__ */ new Set([
-  "lcm.sqlite-shm",
-  "lcm.sqlite-wal"
-]);
-function shouldPreferIncomingOfflineRuntimeFile(relPosix) {
-  const parts = relPosix.split("/");
-  const basename = parts[parts.length - 1] ?? "";
-  return isCanonicalRuntimeStatePath(parts) && REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES.has(basename);
-}
-function shouldDeleteAbsentIncomingOfflineRuntimeFile(relPosix) {
-  const parts = relPosix.split("/");
-  const basename = parts[parts.length - 1] ?? "";
-  return isCanonicalRuntimeStatePath(parts) && ABSENT_INCOMING_RUNTIME_DELETE_FILES.has(basename);
-}
 function filterBaseFilesForMode(files, includeTranscripts) {
   return files.filter((file) => !shouldExcludeRelPath(file.path, includeTranscripts));
 }
@@ -526,7 +467,14 @@ async function* iterateOfflineSyncSnapshotFileRecords(options) {
   const rootAbs = path.resolve(options.root);
   const root = await prepareSafeArchiveRoot(rootAbs, "iterateOfflineSyncSnapshotFileRecords", "root");
   const includeTranscripts = options.includeTranscripts !== false;
-  async function* walk(dirAbs) {
+  const pushExcludes = options.excludeNodeLocalState !== false;
+  const skipGenerationPaths = options.skipEmbeddingGenerationPaths ?? (pushExcludes ? (await computeOmittedEmbeddingGenerationPaths({
+    rootAbs: root.abs,
+    userExcludeRegexps: options.userExcludeRegexps,
+    excludeFile: options.excludeFile,
+    isExcludedRelPath: (relPath) => shouldExcludePushRelPath(relPath, includeTranscripts, options.userExcludeRegexps)
+  })).omittedPaths : void 0);
+  async function* walk(dirAbs, includeContent = options.includeContent === true) {
     throwIfOfflineSyncAborted(options.signal);
     let entries = await readdir(dirAbs, { withFileTypes: true });
     entries = entries.sort((left, right) => left.name.localeCompare(right.name));
@@ -534,9 +482,25 @@ async function* iterateOfflineSyncSnapshotFileRecords(options) {
       const abs = path.join(dirAbs, entry.name);
       const relPosix = path.relative(root.abs, abs).split(path.sep).join("/");
       if (options.excludeNodeLocalState === false ? shouldExcludeRelPath(relPosix, includeTranscripts) : shouldExcludePushRelPath(relPosix, includeTranscripts, options.userExcludeRegexps)) continue;
+      if (skipGenerationPaths?.has(relPosix)) continue;
       if (entry.isSymbolicLink()) continue;
       if (entry.isDirectory()) {
-        yield* walk(abs);
+        if (isEmbeddingGenerationDirPath(relPosix)) {
+          yield* withEmbeddingGenerationLockIter(
+            path.dirname(abs),
+            (content) => walk(abs, content),
+            includeContent,
+            () => assertEmbeddingGenerationStillIncluded(
+              root.abs,
+              relPosix,
+              options.excludeFile,
+              (rel) => shouldExcludePushRelPath(rel, includeTranscripts, options.userExcludeRegexps),
+              []
+            )
+          );
+          continue;
+        }
+        yield* walk(abs, includeContent);
         continue;
       }
       if (!entry.isFile()) continue;
@@ -549,7 +513,7 @@ async function* iterateOfflineSyncSnapshotFileRecords(options) {
         root,
         relPath: relPosix,
         filePath: abs,
-        includeContent: options.includeContent === true,
+        includeContent,
         readFile: options.readFile,
         readFileDigest: options.readFileDigest,
         signal: options.signal
@@ -561,8 +525,20 @@ async function* iterateOfflineSyncSnapshotFileRecords(options) {
 async function buildOfflineSyncSnapshot(options) {
   throwIfOfflineSyncAborted(options.signal);
   const includeTranscripts = options.includeTranscripts !== false;
+  const pushExcludes = options.excludeNodeLocalState !== false;
+  const normalizedDeletions = normalizeDeletionRevisions(options.deletions, "deletions");
+  const omission = pushExcludes ? await computeOmittedEmbeddingGenerationPaths({
+    rootAbs: path.resolve(options.root),
+    userExcludeRegexps: options.userExcludeRegexps,
+    excludeFile: options.excludeFile,
+    tombstonedPaths: (normalizedDeletions ?? []).map((deletion) => deletion.path),
+    isExcludedRelPath: (relPath) => shouldExcludePushRelPath(relPath, includeTranscripts, options.userExcludeRegexps)
+  }) : void 0;
   const files = [];
-  for await (const file of iterateOfflineSyncSnapshotFileRecords(options)) files.push(file);
+  for await (const file of iterateOfflineSyncSnapshotFileRecords({
+    ...options,
+    skipEmbeddingGenerationPaths: omission?.omittedPaths
+  })) files.push(file);
   throwIfOfflineSyncAborted(options.signal);
   const sortedFiles = files.sort(compareByPath);
   const deletions = snapshotBuilderDeletions({
@@ -570,7 +546,8 @@ async function buildOfflineSyncSnapshot(options) {
     files: sortedFiles,
     includeTranscripts,
     excludeNodeLocalState: options.excludeNodeLocalState,
-    userExcludeRegexps: options.userExcludeRegexps
+    userExcludeRegexps: options.userExcludeRegexps,
+    omittedGenerationDirs: omission && omission.omittedDirs.length > 0 ? new Set(omission.omittedDirs) : void 0
   });
   return {
     format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
@@ -579,7 +556,8 @@ async function buildOfflineSyncSnapshot(options) {
     sourceId: normalizeSourceId(options.sourceId, "sourceId"),
     includeTranscripts,
     files: sortedFiles,
-    ...deletions === void 0 ? {} : { deletions }
+    ...deletions === void 0 ? {} : { deletions },
+    ...omission && omission.omittedDirs.length > 0 ? { omittedEmbeddingGenerationDirs: omission.omittedDirs } : {}
   };
 }
 async function buildOfflineSyncSnapshotFromBase(options) {
@@ -593,6 +571,15 @@ async function buildOfflineSyncSnapshotFromBase(options) {
   ));
   const rawBaseCapturedAtMs = options.baseCapturedAt?.getTime();
   const baseCapturedAtMs = rawBaseCapturedAtMs !== void 0 && Number.isFinite(rawBaseCapturedAtMs) ? rawBaseCapturedAtMs : null;
+  const pushExcludes = options.excludeNodeLocalState !== false;
+  const normalizedDeletions = normalizeDeletionRevisions(options.deletions, "deletions");
+  const omission = pushExcludes ? await computeOmittedEmbeddingGenerationPaths({
+    rootAbs: root.abs,
+    userExcludeRegexps: options.userExcludeRegexps,
+    excludeFile: options.excludeFile,
+    tombstonedPaths: (normalizedDeletions ?? []).map((deletion) => deletion.path),
+    isExcludedRelPath: (relPath) => shouldExcludePushRelPath(relPath, includeTranscripts, options.userExcludeRegexps)
+  }) : void 0;
   const files = [];
   async function walk(dirAbs) {
     throwIfOfflineSyncAborted(options.signal);
@@ -603,8 +590,22 @@ async function buildOfflineSyncSnapshotFromBase(options) {
       const abs = path.join(dirAbs, entry.name);
       const relPosix = path.relative(root.abs, abs).split(path.sep).join("/");
       if (options.excludeNodeLocalState === false ? shouldExcludeRelPath(relPosix, includeTranscripts) : shouldExcludePushRelPath(relPosix, includeTranscripts, options.userExcludeRegexps)) continue;
+      if (omission?.omittedPaths.has(relPosix)) continue;
       if (entry.isSymbolicLink()) continue;
       if (entry.isDirectory()) {
+        if (isEmbeddingGenerationDirPath(relPosix)) {
+          await withEmbeddingGenerationLock(path.dirname(abs), async () => {
+            await assertEmbeddingGenerationStillIncluded(
+              root.abs,
+              relPosix,
+              options.excludeFile,
+              (rel) => shouldExcludePushRelPath(rel, includeTranscripts, options.userExcludeRegexps),
+              (normalizedDeletions ?? []).map((deletion) => deletion.path)
+            );
+            await walk(abs);
+          });
+          continue;
+        }
         await walk(abs);
         continue;
       }
@@ -639,7 +640,8 @@ async function buildOfflineSyncSnapshotFromBase(options) {
     files: sortedFiles,
     includeTranscripts,
     excludeNodeLocalState: options.excludeNodeLocalState,
-    userExcludeRegexps: options.userExcludeRegexps
+    userExcludeRegexps: options.userExcludeRegexps,
+    omittedGenerationDirs: omission && omission.omittedDirs.length > 0 ? new Set(omission.omittedDirs) : void 0
   });
   return {
     format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
@@ -648,7 +650,8 @@ async function buildOfflineSyncSnapshotFromBase(options) {
     sourceId: normalizeSourceId(options.sourceId, "sourceId"),
     includeTranscripts,
     files: sortedFiles,
-    ...deletions === void 0 ? {} : { deletions }
+    ...deletions === void 0 ? {} : { deletions },
+    ...omission && omission.omittedDirs.length > 0 ? { omittedEmbeddingGenerationDirs: omission.omittedDirs } : {}
   };
 }
 async function buildOfflineSyncSnapshotForPaths(options) {
@@ -942,8 +945,9 @@ async function applyOfflineSyncSnapshot(options) {
   ));
   const incomingMap = byPath(snapshot.files);
   const deletionMtimeByPath = snapshot.deletions === void 0 ? void 0 : new Map(snapshot.deletions.map((deletion) => [deletion.path, deletion.mtimeMs]));
-  const incomingBuffers = verifyRecordContents(snapshot.files, "offline sync snapshot", {
-    requireContent: false
+  const incomingBuffers = await verifyRecordContents(snapshot.files, "offline sync snapshot", {
+    requireContent: false,
+    readIncomingFile: options.readIncomingFile
   });
   const root = await ensureSyncRoot(options.root, "applyOfflineSyncSnapshot");
   const currentFiles = options.currentFiles ? filterBaseFilesForMode(normalizeFileStates(options.currentFiles), snapshot.includeTranscripts).sort(compareByPath) : (await buildOfflineSyncSnapshot({
@@ -956,10 +960,18 @@ async function applyOfflineSyncSnapshot(options) {
     excludeNodeLocalState: false
   })).files;
   const currentMap = byPath(currentFiles);
-  const deferredPaths = new Set(options.deferredPaths ?? []);
+  const deferredPaths = /* @__PURE__ */ new Set([
+    ...options.deferredPaths ?? [],
+    ...divergedEmbeddingGenerationDeferrals({
+      incomingFiles: snapshot.files,
+      baseFiles: [...baseMap.values()],
+      currentFiles
+    })
+  ]);
+  const omittedGenerationDirs = new Set(snapshot.omittedEmbeddingGenerationDirs ?? []);
   if (deletionMtimeByPath && options.recordDeletionRevision) {
     for (const [relPath, mtimeMs] of deletionMtimeByPath) {
-      if (currentMap.has(relPath) || deferredPaths.has(relPath) || matchesOfflineSyncDefaultExclude(relPath)) {
+      if (currentMap.has(relPath) || deferredPaths.has(relPath) || omittedGenerationDirs.size > 0 && isPathInOmittedEmbeddingGeneration(relPath, omittedGenerationDirs) || matchesOfflineSyncDefaultExclude(relPath)) {
         continue;
       }
       await options.recordDeletionRevision({
@@ -982,6 +994,133 @@ async function applyOfflineSyncSnapshot(options) {
     if (buffer || options.allowMissingConflictContent === true) return buffer;
     return requiredBuffer(incomingBuffers, relPath);
   };
+  const incomingGenerations = detectIncomingEmbeddingGenerations(incomingMap.keys());
+  const generationDirs = /* @__PURE__ */ new Set([
+    ...incomingGenerations.shardDirs,
+    ...incomingGenerations.legacyMarkerDirs
+  ]);
+  const deferredGenerationDirs = /* @__PURE__ */ new Set();
+  for (const deferredPath of deferredPaths) {
+    const membership = embeddingGenerationMembership(deferredPath);
+    if (membership) {
+      deferredGenerationDirs.add(membership.shardDir);
+    }
+  }
+  const transactionResults = [];
+  for (const shardDir of generationDirs) {
+    if (omittedGenerationDirs.has(shardDir)) {
+      throw new EmbeddingIndexStorageError(
+        `offline sync snapshot both includes and omits embedding generation ${shardDir}; refusing to apply`
+      );
+    }
+  }
+  if (generationDirs.size > 0 && (options.writeFile || options.readFile || options.readFileDigest || options.deleteFile) && !(options.writeStagingFile && options.readStagingFile)) {
+    throw new EmbeddingIndexStorageError(
+      "refusing to replace embedding generation: custom storage IO requires both writeStagingFile and readStagingFile hooks; unencrypted raw staging would plaintext-downgrade the index"
+    );
+  }
+  for (const shardDir of generationDirs) {
+    if (deferredGenerationDirs.has(shardDir)) continue;
+    if (incomingGenerations.legacyMarkerDirs.has(shardDir) && !incomingGenerations.shardDirs.has(shardDir)) {
+      const markerRel = [...incomingMap.keys()].find(
+        (relPath) => embeddingMarkerDirOf(relPath) === shardDir.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length)
+      );
+      if (!markerRel) continue;
+      transactionResults.push(await applyEmbeddingGenerationTransaction({
+        root,
+        shardDirRel: shardDir,
+        incomingShardPaths: [],
+        incomingMarker: {
+          path: markerRel,
+          sha256: incomingMap.get(markerRel).sha256,
+          buffer: incomingBuffers.get(markerRel) ?? null
+        },
+        incomingMarkerPresent: false,
+        incomingShardStates: /* @__PURE__ */ new Map(),
+        incomingBuffers,
+        baseStates: baseMap,
+        io: {
+          readFile: options.readFile,
+          writeStagingFile: options.writeStagingFile,
+          readStagingFile: options.readStagingFile,
+          deleteFile: options.deleteFile
+        },
+        now: Date.now()
+      }));
+      continue;
+    }
+    transactionResults.push(await applyEmbeddingGenerationTransaction({
+      root,
+      shardDirRel: shardDir,
+      incomingShardPaths: [...incomingMap.keys()].filter((relPath) => embeddingShardDirOf(relPath) === shardDir).sort(),
+      incomingMarker: null,
+      incomingMarkerPresent: Boolean(
+        incomingGenerations.legacyMarkerDirs.has(shardDir)
+      ),
+      incomingShardStates: new Map(
+        [...incomingMap.values()].filter((relPath) => embeddingShardDirOf(relPath.path) === shardDir).map((state) => [state.path, state])
+      ),
+      incomingBuffers,
+      baseStates: baseMap,
+      io: {
+        readFile: options.readFile,
+        writeStagingFile: options.writeStagingFile,
+        readStagingFile: options.readStagingFile,
+        deleteFile: options.deleteFile
+      },
+      now: Date.now()
+    }));
+  }
+  const localGenerationDirs = /* @__PURE__ */ new Set();
+  for (const relPath of currentMap.keys()) {
+    const membership = embeddingGenerationMembership(relPath);
+    if (membership?.kind === "shard") localGenerationDirs.add(membership.shardDir);
+  }
+  for (const shardDir of localGenerationDirs) {
+    if (generationDirs.has(shardDir) || deferredGenerationDirs.has(shardDir) || omittedGenerationDirs.has(shardDir)) {
+      continue;
+    }
+    const localMembers = [...currentMap.keys()].filter(
+      (relPath) => embeddingGenerationMembership(relPath)?.shardDir === shardDir
+    );
+    if (localMembers.length === 0) continue;
+    const baseMembers = localMembers.filter((relPath) => baseMap.has(relPath));
+    const tombstonedMembers = deletionMtimeByPath === void 0 ? [] : [...deletionMtimeByPath.keys()].filter(
+      (relPath) => embeddingGenerationMembership(relPath)?.shardDir === shardDir
+    );
+    const tombstoneCovered = tombstonedMembers.length > 0;
+    const baseEvidenced = baseMembers.length > 0 && baseMembers.every((relPath) => !incomingMap.has(relPath));
+    if (!tombstoneCovered && !baseEvidenced) continue;
+    transactionResults.push(await applyEmbeddingGenerationRemoval({
+      root,
+      shardDirRel: shardDir,
+      io: { deleteFile: options.deleteFile, readFile: options.readFile },
+      deletionMtimeByPath,
+      baseStates: baseMap,
+      now: Date.now()
+    }));
+  }
+  const transactionHandled = /* @__PURE__ */ new Set();
+  let generationUpserted = 0;
+  let generationDeleted = 0;
+  for (const result of transactionResults) {
+    if (result.deferredLocalDivergence) {
+      for (const relPath of result.handledPaths) deferredPaths.add(relPath);
+      deferredGenerationDirs.add(result.shardDirRel);
+      continue;
+    }
+    generationUpserted += result.upserted;
+    generationDeleted += result.deleted;
+    for (const relPath of result.handledPaths) transactionHandled.add(relPath);
+    for (const [relPath, state] of result.writtenStates) {
+      nextBase.set(relPath, state);
+      transactionHandled.add(relPath);
+    }
+    for (const relPath of result.removedPaths) {
+      nextBase.delete(relPath);
+      transactionHandled.add(relPath);
+    }
+  }
   for (const relPath of unionPaths(baseMap, incomingMap, currentMap)) {
     const base = baseMap.get(relPath);
     const incoming = incomingMap.get(relPath);
@@ -992,6 +1131,14 @@ async function applyOfflineSyncSnapshot(options) {
       skipped += 1;
       continue;
     }
+    if (transactionHandled.has(relPath)) continue;
+    const membership = embeddingGenerationMembership(relPath);
+    if (membership && deferredGenerationDirs.has(membership.shardDir)) {
+      if (base) nextBase.set(relPath, base);
+      else nextBase.delete(relPath);
+      skipped += 1;
+      continue;
+    }
     if (incoming) {
       if (currentEntry?.sha256 === incoming.sha256) {
         if (await setSafeFileMtime(root, relPath, incoming.mtimeMs)) {
@@ -1079,6 +1226,11 @@ async function applyOfflineSyncSnapshot(options) {
       skipped += 1;
       continue;
     }
+    if (shouldPreferIncomingOfflineRuntimeFile(relPath) && membership !== null && omittedGenerationDirs.has(membership.shardDir)) {
+      pendingLocal += 1;
+      skipped += 1;
+      continue;
+    }
     if (shouldPreferIncomingOfflineRuntimeFile(relPath) && (base || shouldDeleteAbsentIncomingOfflineRuntimeFile(relPath))) {
       await deleteSafeFile(
         root,
@@ -1120,8 +1272,8 @@ async function applyOfflineSyncSnapshot(options) {
     skipped += 1;
   }
   return {
-    upserted,
-    deleted,
+    upserted: upserted + generationUpserted,
+    deleted: deleted + generationDeleted,
     skipped,
     pendingLocal,
     conflicts,
@@ -1140,7 +1292,7 @@ async function applyOfflineSyncChangeset(options) {
   }
   const root = await ensureSyncRoot(options.root, "applyOfflineSyncChangeset");
   const records = changeset.changes.filter((change) => change.type === "upsert").map((change) => change.file);
-  const incomingBuffers = verifyRecordContents(records, "offline sync changeset");
+  const incomingBuffers = await verifyRecordContents(records, "offline sync changeset");
   const currentFiles = options.currentFiles ? filterBaseFilesForMode(normalizeFileStates(options.currentFiles), changeset.includeTranscripts).sort(compareByPath) : (await buildOfflineSyncSnapshotForPaths({
     root: root.abs,
     sourceId: "local",
@@ -1251,10 +1403,25 @@ async function applyOfflineSyncChangeset(options) {
     ...options.returnCurrentFiles === false ? { currentFilesComplete: false } : {}
   };
 }
-function verifyRecordContents(records, context, options = {}) {
+async function verifyRecordContents(records, context, options = {}) {
   const buffers = /* @__PURE__ */ new Map();
   for (const record of records) {
     if (typeof record.contentBase64 !== "string") {
+      if (options.readIncomingFile) {
+        const incoming = await options.readIncomingFile({
+          path: record.path,
+          sha256: record.sha256,
+          bytes: record.bytes
+        });
+        if (incoming) {
+          const digest2 = sha256Buffer(incoming);
+          if (digest2.sha256 !== record.sha256 || digest2.bytes !== record.bytes) {
+            throw new Error(`${context}: incoming content checksum mismatch for ${record.path}`);
+          }
+          buffers.set(record.path, incoming);
+          continue;
+        }
+      }
       if (options.requireContent === false) continue;
       throw new Error(`${context}: contentBase64 is required for ${record.path}`);
     }
@@ -1332,25 +1499,6 @@ async function writeSafeFile(root, relPath, content, writeFileHook, mtimeMs) {
     throw error;
   }
 }
-async function setSafeFileMtime(root, relPath, mtimeMs) {
-  if (mtimeMs === void 0) return true;
-  const target = await resolveSafeArchiveTarget(root, relPath);
-  const targetStat = await lstat(target).catch((error) => {
-    if (error.code === "ENOENT") return null;
-    throw error;
-  });
-  if (!targetStat) return false;
-  if (targetStat.isSymbolicLink()) {
-    throw new Error(`offline sync target is a symlink: ${relPath}`);
-  }
-  const targetMtimeMs = assertOfflineSyncMtimeMs(mtimeMs, "mtimeMs");
-  if (Math.abs(targetStat.mtimeMs - targetMtimeMs) <= OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS) {
-    return true;
-  }
-  const mtime = new Date(targetMtimeMs);
-  await utimes(target, mtime, mtime);
-  return true;
-}
 async function readLocalFileState(options) {
   const filePath = await resolveSafeArchiveTarget(
     await prepareSafeArchiveRoot(options.rootAbs, "readLocalFileState", "rootAbs"),
@@ -1562,204 +1710,6 @@ async function applyOfflineSyncFileContentChunk(options) {
     });
   }
 }
-function offlineUploadRelPath(options) {
-  const key = hashText([
-    options.sourceId,
-    options.relPath,
-    options.sha256,
-    String(options.bytes)
-  ].join("\0"));
-  return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
-}
-async function offlineUploadPath(root, options) {
-  const relPath = offlineUploadRelPath(options);
-  return {
-    kind: "single",
-    relPath,
-    filePath: await resolveSafeArchiveTarget(root, relPath)
-  };
-}
-async function offlineUploadChunkPath(root, options) {
-  const uploadRelPath = offlineUploadRelPath(options);
-  const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
-  return {
-    kind: "chunks",
-    relPath,
-    filePath: await resolveSafeArchiveTarget(root, relPath)
-  };
-}
-async function writeOfflineUploadChunk(options) {
-  if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
-    throw new Error("offline sync upload chunk storage hooks require readFile");
-  }
-  const uploadRoot = {
-    ...await offlineUploadPath(options.root, options),
-    kind: "chunks"
-  };
-  if (options.offset === 0) {
-    await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {
-    });
-  } else {
-    const existing = await stat(uploadRoot.filePath).catch((error) => {
-      if (error.code === "ENOENT") return null;
-      throw error;
-    });
-    if (!existing || !existing.isDirectory()) {
-      throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
-    }
-  }
-  const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
-  const writeStagingFile = options.writeStagingFile ?? options.writeFile;
-  if (writeStagingFile) {
-    await writeOfflineUploadContent({
-      root: options.root,
-      relPath: chunk.relPath,
-      filePath: chunk.filePath,
-      content: options.content,
-      writeFile: writeStagingFile
-    });
-    return uploadRoot;
-  }
-  await mkdir(path.dirname(chunk.filePath), { recursive: true });
-  const existingChunk = await lstat(chunk.filePath).catch((error) => {
-    if (error.code === "ENOENT") return null;
-    throw error;
-  });
-  if (existingChunk?.isSymbolicLink()) {
-    throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
-  }
-  await writeFile(chunk.filePath, options.content, { mode: 384 });
-  return uploadRoot;
-}
-async function pruneOfflineUploadStaging(root) {
-  const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
-  const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
-  const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error) => {
-    if (error.code === "ENOENT") return [];
-    throw error;
-  });
-  const now = Date.now();
-  await Promise.all(entries.map(async (entry) => {
-    if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
-    const relPath = `${uploadsRelPath}/${entry.name}`;
-    const filePath = await resolveSafeArchiveTarget(root, relPath);
-    const info = await lstat(filePath).catch((error) => {
-      if (error.code === "ENOENT") return null;
-      throw error;
-    });
-    if (!info) return;
-    if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
-    await rm(filePath, { recursive: true, force: true });
-  }));
-}
-async function* readOfflineUploadStagingChunks(options) {
-  if (options.upload.kind === "single") {
-    yield await readOfflineUploadContent({
-      root: options.root,
-      relPath: options.upload.relPath,
-      filePath: options.upload.filePath,
-      readFile: options.readFile
-    });
-    return;
-  }
-  const entries = await readdir(options.upload.filePath);
-  const chunkNames = entries.filter((entry) => /^\d{20}\.part$/.test(entry)).sort();
-  if (chunkNames.length === 0) {
-    throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
-  }
-  let expectedOffset = 0;
-  for (const chunkName of chunkNames) {
-    const offset = Number(chunkName.slice(0, 20));
-    if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
-      throw new Error(
-        `offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`
-      );
-    }
-    const relPath = `${options.upload.relPath}/${chunkName}`;
-    const filePath = await resolveSafeArchiveTarget(options.root, relPath);
-    const content = await readOfflineUploadContent({
-      root: options.root,
-      relPath,
-      filePath,
-      readFile: options.readFile
-    });
-    expectedOffset += content.length;
-    yield content;
-  }
-}
-async function digestOfflineUploadStagingContent(options) {
-  const hash = createHash("sha256");
-  let bytes = 0;
-  for await (const chunk of readOfflineUploadStagingChunks(options)) {
-    hash.update(chunk);
-    bytes += chunk.length;
-  }
-  return { sha256: hash.digest("hex"), bytes };
-}
-async function writeSafeFileFromUpload(root, relPath, upload, readFile2, writeFileChunks, mtimeMs) {
-  const target = await resolveSafeArchiveTarget(root, relPath);
-  const chunks = readOfflineUploadStagingChunks({ root, upload, readFile: readFile2 });
-  if (writeFileChunks) {
-    await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
-    await setSafeFileMtime(root, relPath, mtimeMs);
-    return;
-  }
-  await mkdir(path.dirname(target), { recursive: true });
-  const tmp = path.join(
-    path.dirname(target),
-    `.remnic-sync.${process.pid}.${randomUUID()}.tmp`
-  );
-  const handle = await open(tmp, "w", 384);
-  try {
-    for await (const chunk of chunks) {
-      if (chunk.length > 0) await handle.write(chunk);
-    }
-    await handle.close();
-    const targetStat = await lstat(target).catch((error) => {
-      if (error.code === "ENOENT") return null;
-      throw error;
-    });
-    if (targetStat?.isSymbolicLink()) {
-      throw new Error(`offline sync target is a symlink: ${relPath}`);
-    }
-    await rename(tmp, target);
-    await setSafeFileMtime(root, relPath, mtimeMs);
-  } catch (error) {
-    await handle.close().catch(() => {
-    });
-    await unlink(tmp).catch(() => {
-    });
-    throw error;
-  }
-}
-async function cleanupOfflineUpload(upload) {
-  if (upload.kind === "chunks") {
-    await rm(upload.filePath, { recursive: true, force: true });
-    return;
-  }
-  await unlink(upload.filePath).catch((error) => {
-    if (error.code === "ENOENT") return;
-    throw error;
-  });
-}
-async function readOfflineUploadContent(options) {
-  if (options.readFile) {
-    return options.readFile({
-      root: options.root.abs,
-      path: options.relPath,
-      filePath: options.filePath
-    });
-  }
-  return readFile(options.filePath);
-}
-async function writeOfflineUploadContent(options) {
-  await options.writeFile({
-    root: options.root.abs,
-    path: options.relPath,
-    filePath: options.filePath,
-    content: options.content
-  });
-}
 async function deleteSafeFile(root, relPath, deleteFile, mtimeMs) {
   const target = await resolveSafeArchiveTarget(root, relPath);
   if (deleteFile) {
@@ -1869,14 +1819,10 @@ export {
   OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
   OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
   OFFLINE_SYNC_MAX_MTIME_MS,
-  globToRegExp,
   filterOfflineSyncDeletionRevisions,
   normalizeOfflineSyncSnapshot,
   normalizeOfflineSyncChangeset,
   isInternalRemnicStatePath,
-  compileOfflineSyncExcludeGlobs,
-  parseOfflineSyncExcludes,
-  shouldPreferIncomingOfflineRuntimeFile,
   iterateOfflineSyncSnapshotFileRecords,
   buildOfflineSyncSnapshot,
   buildOfflineSyncSnapshotFromBase,
@@ -1897,4 +1843,4 @@ export {
   normalizeOfflineSyncState,
   fileStatesFromSnapshot
 };
-//# sourceMappingURL=chunk-WQZZDN5M.js.map
+//# sourceMappingURL=chunk-OCFXVDBZ.js.map