@delali/narsil 0.1.15 → 0.2.0

package/dist/{chunk-MEW7HD75.mjs → chunk-XAKNSWIK.mjs}

@@ -1,15 +1,16 @@
-import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, deserializePayloadV1, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, deserializeMetadata, kWayMerge } from './chunk-HMO6OMZ3.mjs';
+import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, validateVectorPromotion, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, tokenize, fanOutQuery, validateRequiredFields, mergePartitionStats, createPartitionIndex, flattenSchema, pruneStatsToQueryTerms, fnv1a, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, deserializeMetadata, kWayMerge } from './chunk-S6IAXOBA.mjs';
 import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
-import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
-import { generateId, createRequestId, isValidWorkerResponse } from './chunk-HRABJLPB.mjs';
+import { ErrorCodes, NarsilError } from './chunk-M2EDG55Y.mjs';
+import { generateId, createRequestId, isValidWorkerResponse } from './chunk-BKAI2WXI.mjs';
 import { encode, decode } from '@msgpack/msgpack';
 
 // src/partitioning/rebalancer.ts
 var CHUNK_SIZE = 1e3;
 function createRebalancer() {
-  let rebalancing = false;
+  const activeManagers = /* @__PURE__ */ new WeakSet();
+  let activeCount = 0;
   async function rebalance(manager, newPartitionCount, router, onProgress) {
-    if (rebalancing) {
+    if (activeManagers.has(manager)) {
       throw new NarsilError(
         ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
         "A rebalance operation is already in progress"
@@ -29,19 +30,14 @@ function createRebalancer() {
         { newPartitionCount, currentPartitionCount: manager.partitionCount }
       );
     }
-    rebalancing = true;
+    activeManagers.add(manager);
+    activeCount++;
     try {
-      const collectedDocs = [];
       const currentPartitions = manager.getAllPartitions();
+      let documentsTotal = 0;
       for (const partition of currentPartitions) {
-        for (const docId of partition.docIds()) {
-          const document = partition.get(docId);
-          if (document) {
-            collectedDocs.push({ docId, document });
-          }
-        }
+        documentsTotal += partition.count();
       }
-      const documentsTotal = collectedDocs.length;
       onProgress?.({
         phase: "scanning",
         documentsProcessed: 0,
@@ -52,24 +48,38 @@ function createRebalancer() {
         newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
       }
       let documentsProcessed = 0;
+      let chunkFill = 0;
       const insertOptions = {
         validate: false,
-        stopWordOverride: manager.config.stopWords,
-        customTokenizer: manager.config.tokenizer,
+        skipClone: true,
+        stopWordOverride: manager.analysis.stopWords,
+        customTokenizer: manager.analysis.customTokenizer,
         collectSurfaces: manager.config.surfaceForms === true
       };
-      for (let chunkStart = 0; chunkStart < collectedDocs.length; chunkStart += CHUNK_SIZE) {
-        const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, collectedDocs.length);
-        for (let i = chunkStart; i < chunkEnd; i++) {
-          const { docId, document } = collectedDocs[i];
+      for (const partition of currentPartitions) {
+        const docIds = [...partition.docIds()];
+        for (const docId of docIds) {
+          const document = partition.getRef(docId);
+          if (!document) continue;
           const targetPartitionId = router.route(docId, newPartitionCount);
           const targetPartition = newPartitions[targetPartitionId];
-          if (targetPartition.has(docId)) {
-            continue;
+          if (!targetPartition.has(docId)) {
+            targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
+            documentsProcessed++;
+          }
+          chunkFill++;
+          if (chunkFill >= CHUNK_SIZE) {
+            chunkFill = 0;
+            onProgress?.({
+              phase: "moving",
+              documentsProcessed,
+              documentsTotal
+            });
+            await new Promise((resolve) => setTimeout(resolve, 0));
           }
-          targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
-          documentsProcessed++;
         }
+      }
+      if (chunkFill > 0) {
         onProgress?.({
           phase: "moving",
           documentsProcessed,
@@ -89,13 +99,14 @@ function createRebalancer() {
         documentsTotal
       });
     } finally {
-      rebalancing = false;
+      activeManagers.delete(manager);
+      activeCount--;
     }
   }
   return {
     rebalance,
     isRebalancing() {
-      return rebalancing;
+      return activeCount > 0;
     }
   };
 }
@@ -293,6 +304,221 @@ function createExecutionPromoter(config) {
   return { check, markPromoted, isPromoted };
 }
 
+// src/engine/analysis-rebuild.ts
+function yieldToEventLoop() {
+  return new Promise((resolve) => {
+    setTimeout(resolve, 0);
+  });
+}
+function toError(value) {
+  return value instanceof Error ? value : new Error(String(value));
+}
+function wireAnalysisRebuild(wiring) {
+  return createAnalysisRebuildCoordinator(wiring.config, {
+    getManager: wiring.getManager,
+    desyncIndex: wiring.desyncIndex,
+    resyncIndex: wiring.resyncIndex,
+    async persistAnalysisRevision(indexName) {
+      const durability = wiring.durabilityManager;
+      if (durability === null) return;
+      await durability.persistMetadata(indexName);
+      await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
+    },
+    emit(payload) {
+      const handlers = wiring.eventHandlers.get("analysisRebuild");
+      if (!handlers) return;
+      for (const handler of handlers) {
+        try {
+          handler(payload);
+        } catch (err) {
+          console.warn("analysisRebuild handler error:", err instanceof Error ? err.message : String(err));
+        }
+      }
+    }
+  });
+}
+function createAnalysisRebuildCoordinator(config, deps) {
+  const stale = /* @__PURE__ */ new Map();
+  const rebuildMode = config?.rebuild ?? "auto";
+  let queue = Promise.resolve();
+  async function rebuildNow(indexName, entry) {
+    const manager = deps.getManager(indexName);
+    if (manager === void 0) {
+      stale.delete(indexName);
+      return;
+    }
+    const partitionCount = manager.partitionCount;
+    deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
+    const wasPromoted = deps.desyncIndex(indexName);
+    try {
+      for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
+        manager.rebuildTextIndex(partitionId);
+        entry.partitionsRebuilt = partitionId + 1;
+        await yieldToEventLoop();
+      }
+      stale.delete(indexName);
+      await deps.persistAnalysisRevision(indexName);
+      await deps.resyncIndex(indexName, wasPromoted);
+      deps.emit({
+        indexName,
+        status: "completed",
+        partitionsRebuilt: entry.partitionsRebuilt,
+        partitionCount: manager.partitionCount
+      });
+    } catch (err) {
+      const error = toError(err);
+      stale.set(indexName, entry);
+      entry.run = null;
+      await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
+      deps.emit({
+        indexName,
+        status: "failed",
+        partitionsRebuilt: entry.partitionsRebuilt,
+        partitionCount,
+        error
+      });
+      throw error;
+    }
+  }
+  function enqueue(indexName, entry) {
+    if (entry.run !== null) {
+      return entry.run;
+    }
+    const run = queue.then(() => rebuildNow(indexName, entry));
+    entry.run = run;
+    queue = run.then(
+      () => void 0,
+      () => void 0
+    );
+    return run;
+  }
+  async function announce(indexName, entry) {
+    const documentCount = deps.getManager(indexName)?.countDocuments() ?? 0;
+    await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
+  }
+  return {
+    markStale(index) {
+      const existing = stale.get(index.indexName);
+      if (existing !== void 0 && existing.run !== null) {
+        return;
+      }
+      stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
+    },
+    clearStale(indexName) {
+      stale.delete(indexName);
+    },
+    isStale(indexName) {
+      return stale.has(indexName);
+    },
+    async reviewStaleIndexes() {
+      for (const [indexName, entry] of [...stale]) {
+        if (entry.run !== null) continue;
+        try {
+          await announce(indexName, entry);
+        } catch (err) {
+          deps.emit({
+            indexName,
+            status: "failed",
+            partitionsRebuilt: 0,
+            partitionCount: deps.getManager(indexName)?.partitionCount ?? 0,
+            error: toError(err)
+          });
+        }
+        if (rebuildMode === "auto") {
+          void enqueue(indexName, entry).catch(() => void 0);
+        }
+      }
+    },
+    rebuild(indexName) {
+      const entry = stale.get(indexName);
+      if (entry === void 0) {
+        return Promise.resolve();
+      }
+      return enqueue(indexName, entry);
+    }
+  };
+}
+
+// src/engine/durability-config.ts
+var WAL_ONLY_FIELDS = ["directory", "mode", "flushIntervalMs", "segmentMaxBytes", "compactionThreshold"];
+function configError(message) {
+  return new NarsilError(ErrorCodes.CONFIG_INVALID, message);
+}
+function requireKnob(name, value, minimum) {
+  if (value === void 0) {
+    return;
+  }
+  if (typeof value !== "number" || !Number.isFinite(value) || value < minimum) {
+    throw configError(`durability.${name} must be a finite number of at least ${minimum}, got ${String(value)}`);
+  }
+}
+function requireValidDurabilityConfig(durability) {
+  if (durability.tier !== void 0 && durability.tier !== "wal" && durability.tier !== "snapshot") {
+    throw configError(`durability.tier must be "wal" or "snapshot", got "${String(durability.tier)}"`);
+  }
+  if (durability.mode !== void 0 && durability.mode !== "sync" && durability.mode !== "async") {
+    throw configError(`durability.mode must be "sync" or "async", got "${String(durability.mode)}"`);
+  }
+  requireKnob("checkpointIntervalMs", durability.checkpointIntervalMs, 0);
+  requireKnob("checkpointMutationThreshold", durability.checkpointMutationThreshold, 1);
+  requireKnob("flushIntervalMs", durability.flushIntervalMs, 0);
+  requireKnob("segmentMaxBytes", durability.segmentMaxBytes, 1);
+  requireKnob("compactionThreshold", durability.compactionThreshold, 1);
+  if (durability.tier === "snapshot") {
+    for (const field of WAL_ONLY_FIELDS) {
+      if (durability[field] !== void 0) {
+        throw configError(
+          `durability.${field} applies to the write-ahead log tier only and cannot combine with durability.tier "snapshot"`
+        );
+      }
+    }
+  }
+}
+function filesystemBackedDirectory(config) {
+  const adapterDirectory = config.persistence?.directory;
+  if (adapterDirectory !== void 0 && adapterDirectory.trim().length > 0) {
+    return adapterDirectory;
+  }
+  return null;
+}
+function resolveDurabilityTier(config) {
+  const filesystemDirectory = filesystemBackedDirectory(config);
+  if (config.durability) {
+    requireValidDurabilityConfig(config.durability);
+  }
+  if (config.durability?.tier === "snapshot") {
+    if (config.persistence === void 0) {
+      throw configError(
+        "Snapshot durability persists through a persistence adapter. Configure persistence, or remove durability.tier"
+      );
+    }
+    return { kind: "snapshot", config: { ...config.durability }, adapter: config.persistence };
+  }
+  if (config.durability) {
+    if (config.durability.directory !== void 0 && config.durability.directory.trim().length > 0) {
+      return { kind: "wal", config: { ...config.durability } };
+    }
+    if (filesystemDirectory !== null) {
+      return { kind: "wal", config: { ...config.durability, directory: filesystemDirectory } };
+    }
+    if (config.persistence !== void 0) {
+      throw configError(
+        "WAL durability requires a filesystem directory, but the configured persistence adapter is not filesystem-backed. Use a filesystem persistence adapter or set durability.directory"
+      );
+    }
+    throw configError(
+      "Durability requires a directory. Set durability.directory explicitly, or configure a filesystem persistence adapter"
+    );
+  }
+  if (config.persistence === void 0) {
+    return null;
+  }
+  if (filesystemDirectory !== null) {
+    return { kind: "wal", config: { directory: filesystemDirectory } };
+  }
+  return { kind: "snapshot", config: {}, adapter: config.persistence };
+}
+
 // src/persistence/durability/node-fs.ts
 var cachedFs = null;
 var cachedPath = null;
@@ -597,7 +823,7 @@ async function spawnWorker() {
     return null;
   }
 }
-function yieldToEventLoop() {
+function yieldToEventLoop2() {
   if (typeof setImmediate === "function") {
     return new Promise((resolve) => setImmediate(resolve));
   }
@@ -609,7 +835,7 @@ async function chunkedChecksum(payload) {
     const end = Math.min(offset + YIELD_CHUNK_BYTES, payload.length);
     state = crc32Update(state, payload.subarray(offset, end));
     if (end < payload.length) {
-      await yieldToEventLoop();
+      await yieldToEventLoop2();
     }
   }
   return crc32Final(state);
@@ -978,6 +1204,10 @@ async function encodeSnapshotBundle(bundle) {
     version: 1,
     schema: bundle.schema,
     language: bundle.language,
+    ...bundle.analysisRevision !== void 0 ? { analysis_revision: bundle.analysisRevision } : {},
+    ...bundle.tokenizer !== void 0 ? { tokenizer: bundle.tokenizer } : {},
+    ...bundle.stopWords !== void 0 ? { stop_words: bundle.stopWords } : {},
+    ...bundle.stopWordList !== void 0 ? { stop_word_list: bundle.stopWordList } : {},
     partitions: bundle.partitions,
     vectorIndexes: bundle.vectorIndexes,
     checkpoint: bundle.checkpoint.map((c) => ({
@@ -1007,15 +1237,38 @@ async function decodeSnapshotBundle(data) {
   if (!Array.isArray(raw.partitions)) {
     throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle missing partitions");
   }
+  if (raw.analysis_revision !== void 0 && typeof raw.analysis_revision !== "string") {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle analysis_revision must be a revision");
+  }
+  if (raw.tokenizer !== void 0 && typeof raw.tokenizer !== "string") {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle tokenizer must be a name");
+  }
+  if (raw.stop_words !== void 0 && typeof raw.stop_words !== "string") {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle stop_words must be a name");
+  }
+  const stopWordList = normalizeStopWordList(raw.stop_word_list);
   return {
     version: 1,
     schema: raw.schema,
     language: raw.language,
+    ...typeof raw.analysis_revision === "string" ? { analysisRevision: raw.analysis_revision } : {},
+    ...typeof raw.tokenizer === "string" ? { tokenizer: raw.tokenizer } : {},
+    ...typeof raw.stop_words === "string" ? { stopWords: raw.stop_words } : {},
+    ...stopWordList !== void 0 ? { stopWordList } : {},
     partitions: raw.partitions,
     vectorIndexes: raw.vectorIndexes ?? {},
     checkpoint: normalizeCheckpoint(raw.checkpoint)
   };
 }
+function normalizeStopWordList(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!Array.isArray(raw) || !raw.every((word) => typeof word === "string")) {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Snapshot bundle stop_word_list must be a list of words");
+  }
+  return raw;
+}
 function normalizeCheckpoint(raw) {
   if (!Array.isArray(raw)) {
     return [];
@@ -1064,6 +1317,10 @@ async function buildSnapshotBundleBytes(input) {
     version: 1,
     schema: input.schema,
     language: input.language,
+    ...input.analysisRevision !== void 0 ? { analysisRevision: input.analysisRevision } : {},
+    ...input.tokenizer !== void 0 ? { tokenizer: input.tokenizer } : {},
+    ...input.stopWords !== void 0 ? { stopWords: input.stopWords } : {},
+    ...input.stopWordList !== void 0 ? { stopWordList: input.stopWordList } : {},
     partitions: partitionBuffers,
     vectorIndexes: vectorPayloads,
     checkpoint
@@ -1104,6 +1361,36 @@ async function truncateCoveredSegments(directory, indexName, checkpoint) {
     }
   }
 }
+async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
+  for (const [partitionId, partition] of [...partitions]) {
+    if (partitionId < partitionCount) continue;
+    try {
+      await partition.walWriter.close();
+    } catch (err) {
+      onCloseError(err instanceof Error ? err : new Error(String(err)));
+    }
+    partitions.delete(partitionId);
+  }
+  const walRoot = `${indexName}/wal/`;
+  for (const key of await directory.list(walRoot)) {
+    const partitionId = walKeyPartitionId(key, walRoot);
+    if (partitionId === null || partitionId < partitionCount) continue;
+    await directory.remove(key);
+  }
+}
+function walKeyPartitionId(key, walRoot) {
+  const tail = key.slice(walRoot.length);
+  const separator = tail.indexOf("/");
+  if (separator <= 0) {
+    return null;
+  }
+  const segment = tail.slice(0, separator);
+  if (!/^\d+$/.test(segment)) {
+    return null;
+  }
+  const value = Number.parseInt(segment, 10);
+  return Number.isSafeInteger(value) ? value : null;
+}
 function parseSegmentStartSeqNo(key, prefix) {
   if (!key.startsWith(prefix)) {
     return null;
@@ -1253,454 +1540,198 @@ function terminateCheckpointWorker() {
   }
 }
 
-// src/engine/vector-coordinator.ts
-function extractVectorFromDoc(document, fieldPath) {
-  const value = getNestedValue(document, fieldPath);
-  if (value === void 0 || value === null) return null;
-  if (value instanceof Float32Array) return value;
-  if (!Array.isArray(value)) return null;
-  const arr = new Float32Array(value.length);
-  for (let i = 0; i < value.length; i++) {
-    const n = value[i];
-    if (typeof n !== "number" || !Number.isFinite(n)) return null;
-    arr[i] = n;
-  }
-  return arr;
+// src/persistence/durability/segment/layout.ts
+var DEFAULT_COMPACTION_THRESHOLD = 12;
+var SEGMENT_ID_WIDTH = 16;
+function manifestKey(indexName) {
+  return `${indexName}/manifest`;
 }
-function deleteNestedValue(obj, path) {
-  if (!path.includes(".")) {
-    delete obj[path];
-    return;
+function legacySnapshotKey(indexName) {
+  return `${indexName}/snapshot`;
+}
+function segmentPrefix(indexName, partitionId) {
+  return `${indexName}/segments/${partitionId}/`;
+}
+function segmentKey(indexName, partitionId, segmentId) {
+  return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
+}
+function vectorSegmentKey(indexName, partitionId, fieldPath, generation) {
+  return `${segmentPrefix(indexName, partitionId)}vec-${encodeFieldPath(fieldPath)}-g${generation}`;
+}
+function formatSegmentId(segmentId) {
+  if (!Number.isInteger(segmentId) || segmentId < 0) {
+    throw new NarsilError(
+      ErrorCodes.PERSISTENCE_SAVE_FAILED,
+      `Segment id ${segmentId} must be a non-negative integer`,
+      { segmentId }
+    );
   }
-  const segments = path.split(".");
-  let current = obj;
-  for (let i = 0; i < segments.length - 1; i++) {
-    const next = current[segments[i]];
-    if (next === null || next === void 0 || typeof next !== "object") return;
-    current = next;
+  return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
+}
+var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
+function encodeFieldPath(fieldPath) {
+  if (!FIELD_PATH_PATTERN.test(fieldPath)) {
+    throw new NarsilError(
+      ErrorCodes.PERSISTENCE_SAVE_FAILED,
+      `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
+      { fieldPath }
+    );
   }
-  delete current[segments[segments.length - 1]];
+  const fingerprint = fnv1a(fieldPath).toString(36);
+  return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
 }
-function vectorsEqual(a, b) {
-  if (!a) return false;
-  if (a.length !== b.length) return false;
-  for (let i = 0; i < a.length; i++) {
-    if (a[i] !== b[i]) return false;
+async function decodeSegmentManifest(data) {
+  const { payloadBytes } = await unpackEnvelopeBytes(data);
+  const raw = decode(payloadBytes);
+  if (raw.version !== SEGMENT_MANIFEST_VERSION) {
+    throw new NarsilError(
+      ErrorCodes.PERSISTENCE_LOAD_FAILED,
+      `Unsupported segment manifest version ${raw.version}; expected ${SEGMENT_MANIFEST_VERSION}`,
+      { version: raw.version }
+    );
   }
-  return true;
+  const base = validateManifestCommon(raw);
+  return {
+    version: SEGMENT_MANIFEST_VERSION,
+    schema: base.schema,
+    language: base.language,
+    checkpoint: normalizeCheckpoint2(raw.checkpoint),
+    partitions: normalizePartitions(raw.partitions)
+  };
 }
-function validateVectorDimensions(vectors, vecIndexes) {
-  for (const [fieldPath, vector] of vectors) {
-    const vecIndex = vecIndexes.get(fieldPath);
-    if (!vecIndex) continue;
-    if (vector.length !== vecIndex.dimension) {
-      throw new NarsilError(
-        ErrorCodes.VECTOR_DIMENSION_MISMATCH,
-        `Vector dimension mismatch for field "${fieldPath}": expected ${vecIndex.dimension}, got ${vector.length}`,
-        { field: fieldPath, expected: vecIndex.dimension, received: vector.length }
-      );
-    }
+function validateManifestCommon(raw) {
+  if (raw.schema === void 0 || typeof raw.schema !== "object") {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing schema");
+  }
+  if (typeof raw.language !== "string") {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing language");
   }
+  return { schema: raw.schema, language: raw.language };
 }
-function prepareDocumentVectors(document, vectorFieldPaths, vecIndexes) {
-  if (vecIndexes.size === 0) {
-    return { partitionDoc: document, extractedVectors: /* @__PURE__ */ new Map() };
+function normalizeCheckpoint2(raw) {
+  if (!Array.isArray(raw)) {
+    return [];
   }
-  const extractedVectors = /* @__PURE__ */ new Map();
-  for (const fieldPath of vectorFieldPaths) {
-    const vec = extractVectorFromDoc(document, fieldPath);
-    if (vec) {
-      extractedVectors.set(fieldPath, vec);
+  const result = [];
+  for (const entry of raw) {
+    if (typeof entry?.partitionId !== "number" || typeof entry.lastSeqNo !== "number" || typeof entry.primaryTerm !== "number") {
+      continue;
     }
+    result.push({ partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm });
   }
-  if (extractedVectors.size === 0) {
-    return { partitionDoc: document, extractedVectors };
-  }
-  const partitionDoc = shallowCopyExcluding(document, extractedVectors.keys());
-  return { partitionDoc, extractedVectors };
+  return result;
 }
-function shallowCopyExcluding(document, fieldPaths) {
-  const topLevelExcludes = /* @__PURE__ */ new Set();
-  const nestedPaths = [];
-  for (const path of fieldPaths) {
-    if (!path.includes(".")) {
-      topLevelExcludes.add(path);
-    } else {
-      nestedPaths.push(path);
-    }
-  }
-  const result = {};
-  for (const key of Object.keys(document)) {
-    if (topLevelExcludes.has(key)) continue;
-    result[key] = document[key];
+function normalizePartitions(raw) {
+  if (!Array.isArray(raw)) {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing partition entries");
   }
-  for (const path of nestedPaths) {
-    const segments = path.split(".");
-    const topKey = segments[0];
-    if (!(topKey in result) || result[topKey] === null || typeof result[topKey] !== "object") continue;
-    const source = document[topKey];
-    let target = { ...source };
-    result[topKey] = target;
-    for (let i = 1; i < segments.length - 1; i++) {
-      const seg = segments[i];
-      const next = target[seg];
-      if (next === null || next === void 0 || typeof next !== "object") break;
-      const copy = { ...next };
-      target[seg] = copy;
-      target = copy;
+  const result = [];
+  for (const entry of raw) {
+    if (!isNonNegativeInteger(entry?.partitionId)) {
+      throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has a partition without a valid id", {
+        partitionId: entry?.partitionId
+      });
     }
-    delete target[segments[segments.length - 1]];
+    const segments = normalizeSegments(entry.segments, entry.partitionId);
+    const nextSegmentId = readNextSegmentId(entry.nextSegmentId, segments, entry.partitionId);
+    result.push({
+      partitionId: entry.partitionId,
+      nextSegmentId,
+      segments,
+      vectors: normalizeVectors(entry.vectors, entry.partitionId)
+    });
   }
   return result;
 }
-function insertDocumentVectors(docId, vectors, vecIndexes) {
-  const insertedFields = [];
-  try {
-    for (const [fieldPath, vector] of vectors) {
-      const vecIndex = vecIndexes.get(fieldPath);
-      if (!vecIndex) continue;
-      vecIndex.insert(docId, vector);
-      insertedFields.push(fieldPath);
-    }
-  } catch (err) {
-    for (const fieldPath of insertedFields) {
-      const vecIndex = vecIndexes.get(fieldPath);
-      if (vecIndex) {
-        vecIndex.remove(docId);
-      }
-    }
-    throw err;
+function normalizeSegments(raw, partitionId) {
+  if (raw === void 0) {
+    return [];
   }
-  return insertedFields;
-}
-function removeDocumentVectors(docId, vecIndexes) {
-  for (const [, vecIndex] of vecIndexes) {
-    vecIndex.remove(docId);
+  if (!Array.isArray(raw)) {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has malformed segment references", {
+      partitionId
+    });
   }
-}
-function updateDocumentVectors(docId, vectors, vecIndexes) {
-  const updatedFields = [];
-  try {
-    for (const [fieldPath, newVec] of vectors) {
-      const vecIndex = vecIndexes.get(fieldPath);
-      if (!vecIndex) continue;
-      if (newVec === null) {
-        if (vecIndex.has(docId)) {
-          const oldVec2 = vecIndex.getVector(docId);
-          vecIndex.remove(docId);
-          updatedFields.push({ fieldPath, oldVec: oldVec2 });
-        }
-        continue;
-      }
-      const oldVec = vecIndex.getVector(docId);
-      if (vectorsEqual(oldVec, newVec)) continue;
-      vecIndex.remove(docId);
-      try {
-        vecIndex.insert(docId, newVec);
-        updatedFields.push({ fieldPath, oldVec });
-      } catch (err) {
-        if (oldVec) {
-          vecIndex.insert(docId, oldVec);
-        }
-        throw err;
-      }
+  if (raw.length > MAX_SEGMENTS_PER_PARTITION) {
+    throw new NarsilError(
+      ErrorCodes.PERSISTENCE_LOAD_FAILED,
+      `Segment manifest lists ${raw.length} segments for partition ${partitionId}, exceeding the maximum of ${MAX_SEGMENTS_PER_PARTITION}`,
+      { partitionId, segmentCount: raw.length, maximum: MAX_SEGMENTS_PER_PARTITION }
+    );
+  }
+  const seenIds = /* @__PURE__ */ new Set();
+  const result = [];
+  for (const segment of raw) {
+    if (!isSegmentId(segment?.id) || !isNonNegativeInteger(segment.docCount) || !isNonNegativeInteger(segment.tombstoneCount) || !isKey(segment.key)) {
+      throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has an invalid segment reference", {
+        partitionId,
+        segment
+      });
     }
-  } catch (err) {
-    for (const { fieldPath, oldVec } of updatedFields) {
-      const vecIndex = vecIndexes.get(fieldPath);
-      if (!vecIndex) continue;
-      vecIndex.remove(docId);
-      if (oldVec) {
-        vecIndex.insert(docId, oldVec);
-      }
+    if (seenIds.has(segment.id)) {
+      throw new NarsilError(
+        ErrorCodes.PERSISTENCE_LOAD_FAILED,
+        `Segment manifest reuses segment id ${segment.id} in partition ${partitionId}`,
+        { partitionId, segmentId: segment.id }
+      );
     }
-    throw err;
+    seenIds.add(segment.id);
+    result.push({
+      id: segment.id,
+      key: segment.key,
+      docCount: segment.docCount,
+      tombstoneCount: segment.tombstoneCount
+    });
   }
+  return result;
 }
-
-// src/distribution/replication/replica.ts
-function validateReplicationEntry(entry, localPrimaryTerm, log) {
-  if (!log.verifyChecksum(entry)) {
-    return { valid: false, error: "REPLICATION_ENTRY_CORRUPT" };
+function readNextSegmentId(value, segments, partitionId) {
+  if (!isSegmentId(value)) {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has an invalid next segment id", {
+      partitionId,
+      nextSegmentId: value
+    });
   }
-  if (entry.primaryTerm < localPrimaryTerm) {
-    return { valid: false, error: "REPLICATION_TERM_MISMATCH" };
+  for (const segment of segments) {
+    if (segment.id >= value) {
+      throw new NarsilError(
+        ErrorCodes.PERSISTENCE_LOAD_FAILED,
+        `Segment manifest next segment id ${value} does not exceed an existing segment id ${segment.id}`,
+        { partitionId, nextSegmentId: value, segmentId: segment.id }
+      );
+    }
   }
-  return { valid: true };
+  return value;
 }
-function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
-  if (entry.document === null) {
-    return;
-  }
-  const document = decode(entry.document);
-  restoreVectorFields(document, vectorFieldPaths);
-  if (manager.has(entry.documentId)) {
-    removeDocumentVectors(entry.documentId, vecIndexes);
-    manager.remove(entry.documentId);
+function normalizeVectors(raw, partitionId) {
+  if (raw === void 0) {
+    return [];
   }
-  const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths, vecIndexes);
-  manager.insert(entry.documentId, partitionDoc);
-  try {
-    insertDocumentVectors(entry.documentId, extractedVectors, vecIndexes);
-  } catch (err) {
-    manager.remove(entry.documentId);
-    throw err;
+  if (!Array.isArray(raw)) {
+    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has malformed vector references", {
+      partitionId
+    });
   }
-  for (const fieldPath of extractedVectors.keys()) {
-    const vecIndex = vecIndexes.get(fieldPath);
-    if (vecIndex) {
-      vecIndex.scheduleBuild();
+  const result = [];
+  for (const vector of raw) {
+    if (typeof vector?.fieldPath !== "string" || !isNonNegativeInteger(vector.generation) || !isKey(vector.key)) {
+      throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has an invalid vector reference", {
+        partitionId,
+        vector
+      });
     }
+    result.push({ fieldPath: vector.fieldPath, generation: vector.generation, key: vector.key });
   }
+  return result;
 }
-function applyDeleteEntry(entry, manager, vecIndexes) {
-  if (!manager.has(entry.documentId)) {
-    return;
-  }
-  removeDocumentVectors(entry.documentId, vecIndexes);
-  manager.remove(entry.documentId);
+function isNonNegativeInteger(value) {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
 }
-function restoreVectorFields(document, vectorFieldPaths) {
-  for (const fieldPath of vectorFieldPaths) {
-    const value = getNestedValue(document, fieldPath);
-    if (value instanceof Uint8Array && !(value instanceof Float32Array)) {
-      if (value.byteLength % 4 !== 0) {
-        throw new Error(
-          `Vector field "${fieldPath}" has invalid byte length ${value.byteLength} (must be divisible by 4)`
-        );
-      }
-      const elementCount = value.byteLength / 4;
-      const float32 = new Float32Array(elementCount);
-      const dataView = new DataView(value.buffer, value.byteOffset, value.byteLength);
-      for (let i = 0; i < elementCount; i += 1) {
-        float32[i] = dataView.getFloat32(i * 4, true);
-      }
-      setNestedValue(document, fieldPath, float32);
-    }
-  }
+function isSegmentId(value) {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_SEGMENT_ID;
 }
-var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
-function setNestedValue(obj, path, value) {
-  if (!path.includes(".")) {
-    if (FORBIDDEN_KEYS.has(path)) {
-      return;
-    }
-    obj[path] = value;
-    return;
-  }
-  const segments = path.split(".");
-  let current = obj;
-  for (let i = 0; i < segments.length - 1; i += 1) {
-    const segment = segments[i];
-    if (segment === void 0 || FORBIDDEN_KEYS.has(segment)) {
-      return;
-    }
-    let next = current[segment];
-    if (next === null || next === void 0 || typeof next !== "object") {
-      next = {};
-      current[segment] = next;
-    }
-    current = next;
-  }
-  const finalSegment = segments.at(-1);
-  if (finalSegment === void 0 || FORBIDDEN_KEYS.has(finalSegment)) {
-    return;
-  }
-  current[finalSegment] = value;
-}
-
-// src/persistence/durability/segment/layout.ts
-var DEFAULT_COMPACTION_THRESHOLD = 12;
-var SEGMENT_ID_WIDTH = 16;
-function manifestKey(indexName) {
-  return `${indexName}/manifest`;
-}
-function legacySnapshotKey(indexName) {
-  return `${indexName}/snapshot`;
-}
-function segmentPrefix(indexName, partitionId) {
-  return `${indexName}/segments/${partitionId}/`;
-}
-function segmentKey(indexName, partitionId, segmentId) {
-  return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
-}
-function vectorSegmentKey(indexName, partitionId, fieldPath, generation) {
-  return `${segmentPrefix(indexName, partitionId)}vec-${encodeFieldPath(fieldPath)}-g${generation}`;
-}
-function formatSegmentId(segmentId) {
-  if (!Number.isInteger(segmentId) || segmentId < 0) {
-    throw new NarsilError(
-      ErrorCodes.PERSISTENCE_SAVE_FAILED,
-      `Segment id ${segmentId} must be a non-negative integer`,
-      { segmentId }
-    );
-  }
-  return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
-}
-var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
-function encodeFieldPath(fieldPath) {
-  if (!FIELD_PATH_PATTERN.test(fieldPath)) {
-    throw new NarsilError(
-      ErrorCodes.PERSISTENCE_SAVE_FAILED,
-      `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
-      { fieldPath }
-    );
-  }
-  const fingerprint = fnv1a(fieldPath).toString(36);
-  return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
-}
-async function decodeSegmentManifest(data) {
-  const { payloadBytes } = await unpackEnvelopeBytes(data);
-  const raw = decode(payloadBytes);
-  if (raw.version !== SEGMENT_MANIFEST_VERSION) {
-    throw new NarsilError(
-      ErrorCodes.PERSISTENCE_LOAD_FAILED,
-      `Unsupported segment manifest version ${raw.version}; expected ${SEGMENT_MANIFEST_VERSION}`,
-      { version: raw.version }
-    );
-  }
-  const base = validateManifestCommon(raw);
-  return {
-    version: SEGMENT_MANIFEST_VERSION,
-    schema: base.schema,
-    language: base.language,
-    checkpoint: normalizeCheckpoint2(raw.checkpoint),
-    partitions: normalizePartitions(raw.partitions)
-  };
-}
-function validateManifestCommon(raw) {
-  if (raw.schema === void 0 || typeof raw.schema !== "object") {
-    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing schema");
-  }
-  if (typeof raw.language !== "string") {
-    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing language");
-  }
-  return { schema: raw.schema, language: raw.language };
-}
-function normalizeCheckpoint2(raw) {
-  if (!Array.isArray(raw)) {
-    return [];
-  }
-  const result = [];
-  for (const entry of raw) {
-    if (typeof entry?.partitionId !== "number" || typeof entry.lastSeqNo !== "number" || typeof entry.primaryTerm !== "number") {
-      continue;
-    }
-    result.push({ partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm });
-  }
-  return result;
-}
-function normalizePartitions(raw) {
-  if (!Array.isArray(raw)) {
-    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest is missing partition entries");
-  }
-  const result = [];
-  for (const entry of raw) {
-    if (!isNonNegativeInteger(entry?.partitionId)) {
-      throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has a partition without a valid id", {
-        partitionId: entry?.partitionId
-      });
-    }
-    const segments = normalizeSegments(entry.segments, entry.partitionId);
-    const nextSegmentId = readNextSegmentId(entry.nextSegmentId, segments, entry.partitionId);
-    result.push({
-      partitionId: entry.partitionId,
-      nextSegmentId,
-      segments,
-      vectors: normalizeVectors(entry.vectors, entry.partitionId)
-    });
-  }
-  return result;
-}
-function normalizeSegments(raw, partitionId) {
-  if (raw === void 0) {
-    return [];
-  }
-  if (!Array.isArray(raw)) {
-    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has malformed segment references", {
-      partitionId
-    });
-  }
-  if (raw.length > MAX_SEGMENTS_PER_PARTITION) {
-    throw new NarsilError(
-      ErrorCodes.PERSISTENCE_LOAD_FAILED,
-      `Segment manifest lists ${raw.length} segments for partition ${partitionId}, exceeding the maximum of ${MAX_SEGMENTS_PER_PARTITION}`,
-      { partitionId, segmentCount: raw.length, maximum: MAX_SEGMENTS_PER_PARTITION }
-    );
-  }
-  const seenIds = /* @__PURE__ */ new Set();
-  const result = [];
-  for (const segment of raw) {
-    if (!isSegmentId(segment?.id) || !isNonNegativeInteger(segment.docCount) || !isNonNegativeInteger(segment.tombstoneCount) || !isKey(segment.key)) {
-      throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has an invalid segment reference", {
-        partitionId,
-        segment
-      });
-    }
-    if (seenIds.has(segment.id)) {
-      throw new NarsilError(
-        ErrorCodes.PERSISTENCE_LOAD_FAILED,
-        `Segment manifest reuses segment id ${segment.id} in partition ${partitionId}`,
-        { partitionId, segmentId: segment.id }
-      );
-    }
-    seenIds.add(segment.id);
-    result.push({
-      id: segment.id,
-      key: segment.key,
-      docCount: segment.docCount,
-      tombstoneCount: segment.tombstoneCount
-    });
-  }
-  return result;
-}
-function readNextSegmentId(value, segments, partitionId) {
-  if (!isSegmentId(value)) {
-    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has an invalid next segment id", {
-      partitionId,
-      nextSegmentId: value
-    });
-  }
-  for (const segment of segments) {
-    if (segment.id >= value) {
-      throw new NarsilError(
-        ErrorCodes.PERSISTENCE_LOAD_FAILED,
-        `Segment manifest next segment id ${value} does not exceed an existing segment id ${segment.id}`,
-        { partitionId, nextSegmentId: value, segmentId: segment.id }
-      );
-    }
-  }
-  return value;
-}
-function normalizeVectors(raw, partitionId) {
-  if (raw === void 0) {
-    return [];
-  }
-  if (!Array.isArray(raw)) {
-    throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has malformed vector references", {
-      partitionId
-    });
-  }
-  const result = [];
-  for (const vector of raw) {
-    if (typeof vector?.fieldPath !== "string" || !isNonNegativeInteger(vector.generation) || !isKey(vector.key)) {
-      throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment manifest has an invalid vector reference", {
-        partitionId,
-        vector
-      });
-    }
-    result.push({ fieldPath: vector.fieldPath, generation: vector.generation, key: vector.key });
-  }
-  return result;
-}
-function isNonNegativeInteger(value) {
-  return typeof value === "number" && Number.isInteger(value) && value >= 0;
-}
-function isSegmentId(value) {
-  return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_SEGMENT_ID;
-}
-function isKey(value) {
-  return typeof value === "string" && value.length > 0 && !value.includes("\0");
+function isKey(value) {
+  return typeof value === "string" && value.length > 0 && !value.includes("\0");
 }
 
 // src/persistence/durability/segment/manifest.ts
@@ -2045,6 +2076,7 @@ async function loadSegmentedSnapshot(directory, indexName, manifest, deps) {
       break;
     }
   }
+  const payloadsByField = /* @__PURE__ */ new Map();
   for (const partition of manifest.partitions) {
     if (partition.partitionId >= deps.manager.partitionCount) {
       throw new NarsilError(
@@ -2053,11 +2085,12 @@ async function loadSegmentedSnapshot(directory, indexName, manifest, deps) {
         { indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
       );
     }
-    await loadPartition(directory, indexName, partition, deps);
+    await loadPartition(directory, indexName, partition, deps, payloadsByField);
   }
+  restoreVectorIndexes(payloadsByField, deps);
   return manifest.checkpoint;
 }
-async function loadPartition(directory, indexName, partition, deps) {
+async function loadPartition(directory, indexName, partition, deps, payloadsByField) {
   const ordered = [];
   for (const segment of partition.segments) {
     ordered.push(await readSegmentContents(directory, segment.key));
@@ -2070,14 +2103,19 @@ async function loadPartition(directory, indexName, partition, deps) {
   });
   deps.manager.deserializePartition(partition.partitionId, merged);
   for (const vector of partition.vectors) {
-    await loadVectorSegment(directory, indexName, partition, vector.fieldPath, vector.key, deps);
+    if (!deps.vectorIndexes.has(vector.fieldPath)) {
+      continue;
+    }
+    const payload = await readVectorSegment(directory, indexName, partition, vector.fieldPath, vector.key);
+    const collected = payloadsByField.get(vector.fieldPath);
+    if (collected === void 0) {
+      payloadsByField.set(vector.fieldPath, [payload]);
+    } else {
+      collected.push(payload);
+    }
   }
 }
-async function loadVectorSegment(directory, indexName, partition, fieldPath, key, deps) {
-  const vecIndex = deps.vectorIndexes.get(fieldPath);
-  if (vecIndex === void 0) {
-    return;
-  }
+async function readVectorSegment(directory, indexName, partition, fieldPath, key) {
   const bytes = await directory.read(key);
   if (bytes === null) {
     throw new NarsilError(
@@ -2086,12 +2124,31 @@ async function loadVectorSegment(directory, indexName, partition, fieldPath, key
       { indexName, partitionId: partition.partitionId, fieldPath, key }
     );
   }
-  vecIndex.deserialize(await decodeVectorSegment(bytes));
-}
-async function decodeVectorSegment(bytes) {
   const { payloadBytes } = await unpackEnvelopeBytes(bytes);
   return decode(payloadBytes);
 }
+function restoreVectorIndexes(payloadsByField, deps) {
+  for (const [fieldPath, payloads] of payloadsByField) {
+    const vecIndex = deps.vectorIndexes.get(fieldPath);
+    if (vecIndex === void 0) {
+      continue;
+    }
+    vecIndex.deserialize(payloads.length === 1 ? payloads[0] : mergePartitionPayloads(payloads));
+  }
+}
+function mergePartitionPayloads(payloads) {
+  const vectors = [];
+  const graphs = [];
+  for (const payload of payloads) {
+    for (const entry of payload.vectors) {
+      vectors.push(entry);
+    }
+    for (const graph of payload.graphs) {
+      graphs.push(graph);
+    }
+  }
+  return { fieldName: payloads[0].fieldName, dimension: payloads[0].dimension, vectors, graphs, sq8: null };
+}
 async function reclaimOrphanedSegments(directory, indexName, manifest) {
   const referenced = manifestReferencedKeys(manifest);
   for (const partition of manifest.partitions) {
@@ -2139,298 +2196,289 @@ function reconstructSchemaFromMetadata(metadata) {
   if (metadata.surfaceForms === true) {
     config.surfaceForms = true;
   }
+  if (metadata.tokenizer !== void 0) {
+    config.tokenizer = metadata.tokenizer;
+  }
+  if (metadata.stopWords !== void 0) {
+    config.stopWords = metadata.stopWords;
+  } else if (metadata.stopWordList !== void 0) {
+    config.stopWords = new Set(metadata.stopWordList);
+  }
+  if (metadata.partitionLimits !== void 0) {
+    config.partitions = metadata.partitionLimits;
+  }
+  if (metadata.defaultScoring !== void 0) {
+    config.defaultScoring = metadata.defaultScoring;
+  }
+  if (metadata.trackPositions !== void 0) {
+    config.trackPositions = metadata.trackPositions;
+  }
+  if (metadata.strict !== void 0) {
+    config.strict = metadata.strict;
+  }
+  if (metadata.required !== void 0) {
+    config.required = metadata.required;
+  }
+  if (metadata.vectorPromotion !== void 0) {
+    config.vectorPromotion = metadata.vectorPromotion;
+  }
   return config;
 }
 
-// src/persistence/durability/segment/build-segment.ts
-function buildSegmentFromEntries(input) {
-  const router = createPartitionRouter();
-  const vectorSink = /* @__PURE__ */ new Map();
-  const manager = createPartitionManager(input.indexName, input.config, input.language, router, 1, vectorSink);
-  const deleted = /* @__PURE__ */ new Set();
-  for (const entry of input.entries) {
-    if (entry.operation === "DELETE") {
-      deleted.add(entry.documentId);
-      applyDeleteEntry(entry, manager, vectorSink);
-    } else {
-      applyIndexEntry(entry, manager, input.vectorFieldPaths, vectorSink);
-    }
+// src/engine/vector-coordinator.ts
+function extractVectorFromDoc(document, fieldPath) {
+  const value = getNestedValue(document, fieldPath);
+  if (value === void 0 || value === null) return null;
+  if (value instanceof Float32Array) return value;
+  if (!Array.isArray(value)) return null;
+  const arr = new Float32Array(value.length);
+  for (let i = 0; i < value.length; i++) {
+    const n = value[i];
+    if (typeof n !== "number" || !Number.isFinite(n)) return null;
+    arr[i] = n;
   }
-  const liveDocIds = collectLiveDocIds(manager);
-  const tombstones = [];
-  for (const docId of deleted) {
-    if (!liveDocIds.has(docId)) {
-      tombstones.push(docId);
-    }
+  return arr;
+}
+function deleteNestedValue(obj, path) {
+  if (!path.includes(".")) {
+    delete obj[path];
+    return;
   }
-  if (liveDocIds.size === 0 && tombstones.length === 0) {
-    return null;
+  const segments = path.split(".");
+  let current = obj;
+  for (let i = 0; i < segments.length - 1; i++) {
+    const next = current[segments[i]];
+    if (next === null || next === void 0 || typeof next !== "object") return;
+    current = next;
   }
-  return {
-    payload: manager.serializePartitionToBytes(0),
-    tombstones,
-    docCount: liveDocIds.size
-  };
-}
-function collectLiveDocIds(manager) {
-  return new Set(manager.getPartition(0).docIds());
+  delete current[segments[segments.length - 1]];
 }
-
-// src/persistence/durability/segment/compaction.ts
-async function compactPartitionSegments(input) {
-  const { segments, compactionThreshold } = input;
-  if (segments.length <= compactionThreshold) {
-    return { segments, nextSegmentId: input.nextSegmentId };
-  }
-  const windowLength = segments.length - compactionThreshold + 1;
-  const start = chooseCheapestWindow(segments, windowLength);
-  const window = segments.slice(start, start + windowLength);
-  const contents = [];
-  for (const ref of window) {
-    contents.push(await readSegmentContents(input.directory, ref.key));
+function vectorsEqual(a, b) {
+  if (!a) return false;
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
   }
-  const merged = mergeTimeOrderedSegments(contents, {
-    indexName: input.indexName,
-    partitionId: input.partitionId,
-    totalPartitions: 1,
-    language: input.language.name
-  });
-  const liveDocIds = new Set(Object.keys(merged.documents));
-  const retainedTombstones = collectRetainedTombstones(contents, liveDocIds);
-  const payload = reserialize(input, merged);
-  const id = input.nextSegmentId;
-  const key = segmentKey(input.indexName, input.partitionId, id);
-  await persistSegmentFile(input.directory, key, payload, retainedTombstones);
-  const mergedRef = {
-    id,
-    key,
-    docCount: liveDocIds.size,
-    tombstoneCount: retainedTombstones.length
-  };
-  const nextSegments = [...segments.slice(0, start), mergedRef, ...segments.slice(start + windowLength)];
-  return { segments: nextSegments, nextSegmentId: id + 1 };
+  return true;
 }
-function chooseCheapestWindow(segments, windowLength) {
-  let prefix = 0;
-  for (let i = 0; i < windowLength; i += 1) {
-    prefix += segments[i].docCount;
-  }
-  let bestStart = 0;
-  let bestCost = prefix;
-  for (let start = 1; start + windowLength <= segments.length; start += 1) {
-    prefix += segments[start + windowLength - 1].docCount - segments[start - 1].docCount;
-    if (prefix < bestCost) {
-      bestCost = prefix;
-      bestStart = start;
+function validateVectorDimensions(vectors, vecIndexes) {
+  for (const [fieldPath, vector] of vectors) {
+    const vecIndex = vecIndexes.get(fieldPath);
+    if (!vecIndex) continue;
+    if (vector.length !== vecIndex.dimension) {
+      throw new NarsilError(
+        ErrorCodes.VECTOR_DIMENSION_MISMATCH,
+        `Vector dimension mismatch for field "${fieldPath}": expected ${vecIndex.dimension}, got ${vector.length}`,
+        { field: fieldPath, expected: vecIndex.dimension, received: vector.length }
+      );
     }
   }
-  return bestStart;
 }
-function collectRetainedTombstones(contents, liveDocIds) {
-  const retained = /* @__PURE__ */ new Set();
-  for (const segment of contents) {
-    for (const docId of segment.tombstones) {
-      if (!liveDocIds.has(docId)) {
-        retained.add(docId);
-      }
+function prepareDocumentVectors(document, vectorFieldPaths, vecIndexes) {
+  if (vecIndexes.size === 0) {
+    return { partitionDoc: document, extractedVectors: /* @__PURE__ */ new Map() };
+  }
+  const extractedVectors = /* @__PURE__ */ new Map();
+  for (const fieldPath of vectorFieldPaths) {
+    const vec = extractVectorFromDoc(document, fieldPath);
+    if (vec) {
+      extractedVectors.set(fieldPath, vec);
     }
   }
-  return [...retained];
-}
-function reserialize(input, merged) {
-  const router = createPartitionRouter();
-  const vectorSink = /* @__PURE__ */ new Map();
-  const manager = createPartitionManager(input.indexName, input.config, input.language, router, 1, vectorSink);
-  manager.deserializePartition(0, merged);
-  return manager.serializePartitionToBytes(0);
-}
-async function writePartitionVectors(input) {
-  if (input.vectorFields.size === 0) {
-    return [];
+  if (extractedVectors.size === 0) {
+    return { partitionDoc: document, extractedVectors };
   }
-  if (input.entries.length === 0) {
-    return input.priorVectors;
+  const partitionDoc = shallowCopyExcluding(document, extractedVectors.keys());
+  return { partitionDoc, extractedVectors };
+}
+function shallowCopyExcluding(document, fieldPaths) {
+  const topLevelExcludes = /* @__PURE__ */ new Set();
+  const nestedPaths = [];
+  for (const path of fieldPaths) {
+    if (!path.includes(".")) {
+      topLevelExcludes.add(path);
+    } else {
+      nestedPaths.push(path);
+    }
   }
-  const priorByField = /* @__PURE__ */ new Map();
-  for (const ref of input.priorVectors) {
-    priorByField.set(ref.fieldPath, ref);
+  const result = {};
+  for (const key of Object.keys(document)) {
+    if (topLevelExcludes.has(key)) continue;
+    result[key] = document[key];
   }
-  const vectorIndexes = /* @__PURE__ */ new Map();
-  for (const [fieldPath, dimension] of input.vectorFields) {
-    vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dimension, input.config.vectorPromotion));
+  for (const path of nestedPaths) {
+    const segments = path.split(".");
+    const topKey = segments[0];
+    if (!(topKey in result) || result[topKey] === null || typeof result[topKey] !== "object") continue;
+    const source = document[topKey];
+    let target = { ...source };
+    result[topKey] = target;
+    for (let i = 1; i < segments.length - 1; i++) {
+      const seg = segments[i];
+      const next = target[seg];
+      if (next === null || next === void 0 || typeof next !== "object") break;
+      const copy = { ...next };
+      target[seg] = copy;
+      target = copy;
+    }
+    delete target[segments[segments.length - 1]];
   }
-  const router = createPartitionRouter();
-  const manager = createPartitionManager(input.indexName, input.config, input.language, router, 1, vectorIndexes);
-  for (const ref of input.priorVectors) {
-    const vecIndex = vectorIndexes.get(ref.fieldPath);
-    if (vecIndex === void 0) {
-      continue;
+  return result;
+}
+function insertDocumentVectors(docId, vectors, vecIndexes) {
+  const insertedFields = [];
+  try {
+    for (const [fieldPath, vector] of vectors) {
+      const vecIndex = vecIndexes.get(fieldPath);
+      if (!vecIndex) continue;
+      vecIndex.insert(docId, vector);
+      insertedFields.push(fieldPath);
     }
-    const bytes = await input.directory.read(ref.key);
-    if (bytes !== null) {
-      const { payloadBytes } = await unpackEnvelopeBytes(bytes);
-      vecIndex.deserialize(decode(payloadBytes));
+  } catch (err) {
+    for (const fieldPath of insertedFields) {
+      const vecIndex = vecIndexes.get(fieldPath);
+      if (vecIndex) {
+        vecIndex.remove(docId);
+      }
     }
+    throw err;
   }
-  for (const entry of input.entries) {
-    if (entry.operation === "DELETE") {
-      applyDeleteEntry(entry, manager, vectorIndexes);
-    } else {
-      applyIndexEntry(entry, manager, input.vectorFieldPaths, vectorIndexes);
-    }
+  return insertedFields;
+}
+function removeDocumentVectors(docId, vecIndexes) {
+  for (const [, vecIndex] of vecIndexes) {
+    vecIndex.remove(docId);
   }
-  const result = [];
-  for (const [fieldPath, vecIndex] of vectorIndexes) {
-    const generation = (priorByField.get(fieldPath)?.generation ?? 0) + 1;
-    const key = vectorSegmentKey(input.indexName, input.partitionId, fieldPath, generation);
-    const parts = await packSnapshotEnvelopePartsRetrying(() => encode(vecIndex.serialize()));
-    await input.directory.atomicWrite(key, [parts.header, parts.payload]);
-    result.push({ fieldPath, generation, key });
+}
+function updateDocumentVectors(docId, vectors, vecIndexes) {
+  const updatedFields = [];
+  try {
+    for (const [fieldPath, newVec] of vectors) {
+      const vecIndex = vecIndexes.get(fieldPath);
+      if (!vecIndex) continue;
+      if (newVec === null) {
+        if (vecIndex.has(docId)) {
+          const oldVec2 = vecIndex.getVector(docId);
+          vecIndex.remove(docId);
+          updatedFields.push({ fieldPath, oldVec: oldVec2 });
+        }
+        continue;
+      }
+      const oldVec = vecIndex.getVector(docId);
+      if (vectorsEqual(oldVec, newVec)) continue;
+      vecIndex.remove(docId);
+      try {
+        vecIndex.insert(docId, newVec);
+        updatedFields.push({ fieldPath, oldVec });
+      } catch (err) {
+        if (oldVec) {
+          vecIndex.insert(docId, oldVec);
+        }
+        throw err;
+      }
+    }
+  } catch (err) {
+    for (const { fieldPath, oldVec } of updatedFields) {
+      const vecIndex = vecIndexes.get(fieldPath);
+      if (!vecIndex) continue;
+      vecIndex.remove(docId);
+      if (oldVec) {
+        vecIndex.insert(docId, oldVec);
+      }
+    }
+    throw err;
   }
-  return result;
 }
 
-// src/persistence/durability/segment/write.ts
-async function writeSegmentedCheckpoint(input) {
-  const { directory, metadata } = input;
-  const indexName = metadata.indexName;
-  const config = reconstructSchemaFromMetadata(metadata);
-  const language = getLanguage(config.language ?? "english");
-  const vectorFields = extractVectorFieldsFromSchema(config.schema);
-  const compactionThreshold = resolveCompactionThreshold(input.compactionThreshold);
-  const context = {
-    directory,
-    indexName,
-    config,
-    language,
-    vectorFields,
-    vectorFieldPaths: new Set(vectorFields.keys()),
-    compactionThreshold
-  };
-  const priorManifest = await readSegmentManifest(directory, indexName);
-  const checkpointByPartition = /* @__PURE__ */ new Map();
-  for (const target of input.targets) {
-    checkpointByPartition.set(target.partitionId, {
-      partitionId: target.partitionId,
-      lastSeqNo: target.lastSeqNo,
-      primaryTerm: target.primaryTerm
-    });
+// src/distribution/replication/replica.ts
+function validateReplicationEntry(entry, localPrimaryTerm, log) {
+  if (!log.verifyChecksum(entry)) {
+    return { valid: false, error: ErrorCodes.REPLICATION_ENTRY_CORRUPT };
   }
-  const partitions = [];
-  for (const target of input.targets) {
-    const priorPartition = priorManifest?.partitions.find((p) => p.partitionId === target.partitionId);
-    const priorSeqNo = snapshotCheckpointFor(priorManifest?.checkpoint ?? [], target.partitionId);
-    const entries = await collectWalEntriesInRange(
-      directory,
-      indexName,
-      target.partitionId,
-      priorSeqNo,
-      target.lastSeqNo
-    );
-    partitions.push(await writePartition(context, target.partitionId, priorPartition, entries));
+  if (entry.primaryTerm < localPrimaryTerm) {
+    return { valid: false, error: ErrorCodes.REPLICATION_TERM_MISMATCH };
   }
-  carryForwardUncheckpointedPartitions(priorManifest, input.targets, partitions, checkpointByPartition);
-  const manifest = {
-    schema: metadata.schema,
-    language: metadata.language,
-    checkpoint: [...checkpointByPartition.values()],
-    partitions
-  };
-  const parts = await encodeSegmentManifest(manifest);
-  await directory.atomicWrite(manifestKey(indexName), [parts.header, parts.payload]);
-  await collectGarbage(directory, indexName, priorManifest, manifest);
+  return { valid: true };
 }
-async function writePartition(context, partitionId, priorPartition, entries) {
-  let segments = priorPartition ? [...priorPartition.segments] : [];
-  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
-  const built = buildSegmentFromEntries({
-    indexName: context.indexName,
-    config: context.config,
-    language: context.language,
-    vectorFieldPaths: context.vectorFieldPaths,
-    entries
-  });
-  if (built !== null) {
-    const id = nextSegmentId;
-    const key = segmentKey(context.indexName, partitionId, id);
-    await persistSegmentFile(context.directory, key, built.payload, built.tombstones);
-    segments.push({ id, key, docCount: built.docCount, tombstoneCount: built.tombstones.length });
-    nextSegmentId = id + 1;
+function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
+  if (entry.document === null) {
+    return;
   }
-  if (segments.length > MAX_SEGMENTS_PER_PARTITION) {
-    throw new NarsilError(
-      ErrorCodes.PERSISTENCE_SAVE_FAILED,
-      `Partition ${partitionId} would exceed the maximum of ${MAX_SEGMENTS_PER_PARTITION} segments`,
-      { partitionId, segmentCount: segments.length, maximum: MAX_SEGMENTS_PER_PARTITION }
-    );
+  const document = decode(entry.document);
+  restoreVectorFields(document, vectorFieldPaths);
+  if (manager.has(entry.documentId)) {
+    removeDocumentVectors(entry.documentId, vecIndexes);
+    manager.remove(entry.documentId);
   }
-  const compacted = await compactPartitionSegments({
-    directory: context.directory,
-    indexName: context.indexName,
-    partitionId,
-    config: context.config,
-    language: context.language,
-    segments,
-    nextSegmentId,
-    compactionThreshold: context.compactionThreshold
-  });
-  segments = compacted.segments;
-  nextSegmentId = compacted.nextSegmentId;
-  const vectors = await writePartitionVectors({
-    directory: context.directory,
-    indexName: context.indexName,
-    partitionId,
-    config: context.config,
-    language: context.language,
-    vectorFields: context.vectorFields,
-    vectorFieldPaths: context.vectorFieldPaths,
-    entries,
-    priorVectors: priorPartition?.vectors ?? []
-  });
-  return { partitionId, nextSegmentId, segments, vectors };
-}
-function carryForwardUncheckpointedPartitions(priorManifest, targets, partitions, checkpointByPartition) {
-  if (priorManifest === null) {
-    return;
+  const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths, vecIndexes);
+  manager.insert(entry.documentId, partitionDoc);
+  try {
+    insertDocumentVectors(entry.documentId, extractedVectors, vecIndexes);
+  } catch (err) {
+    manager.remove(entry.documentId);
+    throw err;
   }
-  const covered = new Set(targets.map((t) => t.partitionId));
-  for (const priorPartition of priorManifest.partitions) {
-    if (covered.has(priorPartition.partitionId)) {
-      continue;
-    }
-    partitions.push(priorPartition);
-    const priorCheckpoint = priorManifest.checkpoint.find((c) => c.partitionId === priorPartition.partitionId);
-    if (priorCheckpoint !== void 0 && !checkpointByPartition.has(priorPartition.partitionId)) {
-      checkpointByPartition.set(priorPartition.partitionId, priorCheckpoint);
+  for (const fieldPath of extractedVectors.keys()) {
+    const vecIndex = vecIndexes.get(fieldPath);
+    if (vecIndex) {
+      vecIndex.scheduleBuild();
     }
   }
 }
-function resolveCompactionThreshold(configured) {
-  if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
-    return DEFAULT_COMPACTION_THRESHOLD;
+function applyDeleteEntry(entry, manager, vecIndexes) {
+  removeDocumentVectors(entry.documentId, vecIndexes);
+  if (!manager.has(entry.documentId)) {
+    return;
   }
-  return configured;
+  manager.remove(entry.documentId);
 }
-async function collectGarbage(directory, indexName, priorManifest, manifest) {
-  const referenced = manifestReferencedKeys(manifest);
-  const partitionIds = /* @__PURE__ */ new Set();
-  for (const partition of manifest.partitions) {
-    partitionIds.add(partition.partitionId);
+function restoreVectorFields(document, vectorFieldPaths) {
+  for (const fieldPath of vectorFieldPaths) {
+    const value = getNestedValue(document, fieldPath);
+    if (value instanceof Uint8Array && !(value instanceof Float32Array)) {
+      if (value.byteLength % 4 !== 0) {
+        throw new Error(
+          `Vector field "${fieldPath}" has invalid byte length ${value.byteLength} (must be divisible by 4)`
+        );
+      }
+      const elementCount = value.byteLength / 4;
+      const float32 = new Float32Array(elementCount);
+      const dataView = new DataView(value.buffer, value.byteOffset, value.byteLength);
+      for (let i = 0; i < elementCount; i += 1) {
+        float32[i] = dataView.getFloat32(i * 4, true);
+      }
+      setNestedValue(document, fieldPath, float32);
+    }
   }
-  if (priorManifest !== null) {
-    for (const partition of priorManifest.partitions) {
-      partitionIds.add(partition.partitionId);
+}
+var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
+function setNestedValue(obj, path, value) {
+  if (!path.includes(".")) {
+    if (FORBIDDEN_KEYS.has(path)) {
+      return;
     }
+    obj[path] = value;
+    return;
   }
-  for (const partitionId of partitionIds) {
-    const prefix = segmentPrefix(indexName, partitionId);
-    for (const key of await directory.list(prefix)) {
-      if (!referenced.has(key)) {
-        await directory.remove(key);
-      }
+  const segments = path.split(".");
+  let current = obj;
+  for (let i = 0; i < segments.length - 1; i += 1) {
+    const segment = segments[i];
+    if (segment === void 0 || FORBIDDEN_KEYS.has(segment)) {
+      return;
+    }
+    let next = current[segment];
+    if (next === null || next === void 0 || typeof next !== "object") {
+      next = {};
+      current[segment] = next;
     }
+    current = next;
   }
-  await directory.remove(legacySnapshotKey(indexName));
+  const finalSegment = segments.at(-1);
+  if (finalSegment === void 0 || FORBIDDEN_KEYS.has(finalSegment)) {
+    return;
+  }
+  current[finalSegment] = value;
 }
 
 // src/distribution/transport/types.ts
@@ -2742,225 +2790,570 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
   return { entries, cleanEnd };
 }
 
-// src/persistence/durability/recovery.ts
-async function listPersistedIndexes(directory) {
-  const metaKeys = await directory.list("");
-  const names = [];
-  for (const key of metaKeys) {
-    if (key.endsWith("/meta")) {
-      names.push(key.slice(0, -"/meta".length));
+// src/persistence/durability/recovery.ts
+async function listPersistedIndexes(directory) {
+  const metaKeys = await directory.list("");
+  const names = [];
+  for (const key of metaKeys) {
+    if (key.endsWith("/meta")) {
+      names.push(key.slice(0, -"/meta".length));
+    }
+  }
+  return names;
+}
+async function loadMetadata(directory, indexName) {
+  const bytes = await directory.read(`${indexName}/meta`);
+  if (bytes === null) {
+    return null;
+  }
+  const { metadata } = await readMetadataEnvelope(bytes);
+  return metadata;
+}
+async function loadSnapshotBundleBytes(bytes, deps) {
+  const bundle = await decodeSnapshotBundle(bytes);
+  while (deps.manager.partitionCount < bundle.partitions.length) {
+    deps.manager.addPartition();
+  }
+  if (bundle.partitions.length > 0) {
+    deps.manager.trimPartitions(bundle.partitions.length);
+  }
+  for (let i = 0; i < bundle.partitions.length; i += 1) {
+    deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
+  }
+  for (const [fieldPath, payload] of Object.entries(bundle.vectorIndexes)) {
+    const vecIndex = deps.vectorIndexes.get(fieldPath);
+    if (vecIndex) {
+      vecIndex.deserialize(payload);
+    }
+  }
+  return bundle.checkpoint;
+}
+async function loadSnapshot(directory, indexName, deps) {
+  const manifest = await readSegmentManifest(directory, indexName);
+  if (manifest !== null) {
+    return loadSegmentedSnapshot(directory, indexName, manifest, deps);
+  }
+  const bytes = await directory.read(`${indexName}/snapshot`);
+  if (bytes === null) {
+    return [];
+  }
+  return loadSnapshotBundleBytes(bytes, deps);
+}
+function segmentStartSeqNo(key, prefix) {
+  if (!key.startsWith(prefix)) {
+    return null;
+  }
+  const tail = key.slice(prefix.length);
+  if (!/^\d{16}$/.test(tail)) {
+    return null;
+  }
+  const value = Number.parseInt(tail, 10);
+  return Number.isSafeInteger(value) ? value : null;
+}
+async function readWalSegments(directory, indexName, partitionId) {
+  const prefix = `${indexName}/wal/${partitionId}/`;
+  const markerBytes = await directory.read(`${prefix}commit`);
+  const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
+  if (marker === null) {
+    return null;
+  }
+  const segments = await collectSegments(directory, prefix);
+  const durableEntries = [];
+  let highestReadFromWal = 0;
+  let activeTail = null;
+  for (const { key, startSeqNo } of segments) {
+    if (startSeqNo > marker.state.activeSegmentSeqNo) {
+      continue;
+    }
+    const bytes = await directory.read(key);
+    if (bytes === null) {
+      continue;
+    }
+    if (startSeqNo < marker.state.activeSegmentSeqNo) {
+      const header = checkSegmentHeader(bytes);
+      if (!header.ok) {
+        throw new NarsilError(
+          ErrorCodes.PERSISTENCE_WAL_CORRUPT,
+          `Sealed WAL segment header invalid: ${header.reason}`,
+          {
+            key,
+            reason: header.reason
+          }
+        );
+      }
+      for (const entry of readDurableRegion(bytes, bytes.length)) {
+        durableEntries.push(entry);
+        if (entry.seqNo > highestReadFromWal) {
+          highestReadFromWal = entry.seqNo;
+        }
+      }
+      continue;
+    }
+    for (const entry of readDurableRegion(bytes, marker.state.durableByteLength)) {
+      durableEntries.push(entry);
+      if (entry.seqNo > highestReadFromWal) {
+        highestReadFromWal = entry.seqNo;
+      }
+    }
+    const tail = readTailBeyondFrontier(
+      bytes,
+      marker.state.durableByteLength,
+      Math.max(highestReadFromWal, marker.state.highestDurableSeqNo)
+    );
+    activeTail = { key, entries: tail.entries, cleanEnd: tail.cleanEnd, segmentLength: bytes.length };
+  }
+  return { marker, segments, durableEntries, activeTail, highestReadFromWal };
+}
+async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
+  const read = await readWalSegments(directory, indexName, partitionId);
+  if (read === null) {
+    return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
+  }
+  const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
+  if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
+    throw new NarsilError(
+      ErrorCodes.PERSISTENCE_WAL_CORRUPT,
+      "A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
+      {
+        indexName,
+        partitionId,
+        highestRead: highestReadFromWal,
+        highestDurable: marker.state.highestDurableSeqNo
+      }
+    );
+  }
+  let highestSeqNo = fromSeqNoExclusive;
+  let highestPrimaryTerm = 0;
+  const replay = (entries) => {
+    for (const entry of entries) {
+      if (entry.seqNo <= highestSeqNo) {
+        continue;
+      }
+      applyEntry(entry, deps);
+      highestSeqNo = entry.seqNo;
+      if (entry.primaryTerm > highestPrimaryTerm) {
+        highestPrimaryTerm = entry.primaryTerm;
+      }
+    }
+  };
+  replay(durableEntries);
+  if (activeTail !== null) {
+    replay(activeTail.entries);
+  }
+  await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
+  if (activeTail !== null && activeTail.cleanEnd < activeTail.segmentLength) {
+    await truncateSegmentTail(directory, activeTail.key, activeTail.cleanEnd);
+  }
+  return { highestSeqNo, highestPrimaryTerm };
+}
+async function collectSegments(directory, prefix) {
+  const keys = await directory.list(prefix);
+  const refs = [];
+  for (const key of keys) {
+    const startSeqNo = segmentStartSeqNo(key, prefix);
+    if (startSeqNo !== null) {
+      refs.push({ key, startSeqNo });
+    }
+  }
+  refs.sort((a, b) => a.startSeqNo - b.startSeqNo);
+  return refs;
+}
+async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
+  for (const { key, startSeqNo } of segments) {
+    if (startSeqNo > activeSegmentSeqNo) {
+      await directory.remove(key);
+    }
+  }
+}
+async function truncateSegmentTail(directory, key, durableByteLength) {
+  const cleanLength = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
+  const handle = await directory.appendHandle(key);
+  try {
+    const currentSize = await handle.size();
+    if (cleanLength < currentSize) {
+      await handle.truncate(cleanLength);
+    }
+  } finally {
+    await handle.close();
+  }
+}
+function applyEntry(entry, deps) {
+  if (entry.operation === "DELETE") {
+    applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
+    return;
+  }
+  applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes);
+}
+function snapshotCheckpointFor(checkpoint, partitionId) {
+  return checkpointLastSeqNo(checkpoint, partitionId);
+}
+async function collectWalEntriesInRange(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive) {
+  const read = await readWalSegments(directory, indexName, partitionId);
+  if (read === null) {
+    return [];
+  }
+  if (Math.max(fromSeqNoExclusive, read.highestReadFromWal) < read.marker.state.highestDurableSeqNo) {
+    throw new NarsilError(
+      ErrorCodes.PERSISTENCE_WAL_CORRUPT,
+      "A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
+      {
+        indexName,
+        partitionId,
+        highestRead: read.highestReadFromWal,
+        highestDurable: read.marker.state.highestDurableSeqNo
+      }
+    );
+  }
+  const entries = [];
+  for (const entry of read.durableEntries) {
+    if (entry.seqNo <= fromSeqNoExclusive || entry.seqNo > upToSeqNoInclusive) {
+      continue;
+    }
+    entries.push(entry);
+  }
+  return entries;
+}
+
+// src/persistence/durability/segment/build-segment.ts
+function buildSegmentFromEntries(input) {
+  const router = createPartitionRouter();
+  const vectorSink = /* @__PURE__ */ new Map();
+  const manager = createPartitionManager(input.indexName, input.config, input.language, router, 1, vectorSink);
+  const deleted = /* @__PURE__ */ new Set();
+  for (const entry of input.entries) {
+    if (entry.operation === "DELETE") {
+      deleted.add(entry.documentId);
+      applyDeleteEntry(entry, manager, vectorSink);
+    } else {
+      applyIndexEntry(entry, manager, input.vectorFieldPaths, vectorSink);
     }
   }
-  return names;
-}
-async function loadMetadata(directory, indexName) {
-  const bytes = await directory.read(`${indexName}/meta`);
-  if (bytes === null) {
+  const liveDocIds = collectLiveDocIds(manager);
+  const tombstones = [];
+  for (const docId of deleted) {
+    if (!liveDocIds.has(docId)) {
+      tombstones.push(docId);
+    }
+  }
+  if (liveDocIds.size === 0 && tombstones.length === 0) {
     return null;
   }
-  const { metadata } = await readMetadataEnvelope(bytes);
-  return metadata;
+  return {
+    payload: manager.serializePartitionToBytes(0),
+    tombstones,
+    docCount: liveDocIds.size
+  };
 }
-async function loadSnapshotBundleBytes(bytes, deps) {
-  const bundle = await decodeSnapshotBundle(bytes);
-  while (deps.manager.partitionCount < bundle.partitions.length) {
-    deps.manager.addPartition();
+function collectLiveDocIds(manager) {
+  return new Set(manager.getPartition(0).docIds());
+}
+
+// src/persistence/durability/segment/compaction.ts
+async function compactPartitionSegments(input) {
+  const { segments, compactionThreshold } = input;
+  if (segments.length <= compactionThreshold) {
+    return { segments, nextSegmentId: input.nextSegmentId };
   }
-  for (let i = 0; i < bundle.partitions.length; i += 1) {
-    deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
+  const windowLength = segments.length - compactionThreshold + 1;
+  const start = chooseCheapestWindow(segments, windowLength);
+  const window = segments.slice(start, start + windowLength);
+  const contents = [];
+  for (const ref of window) {
+    contents.push(await readSegmentContents(input.directory, ref.key));
   }
-  for (const [fieldPath, payload] of Object.entries(bundle.vectorIndexes)) {
-    const vecIndex = deps.vectorIndexes.get(fieldPath);
-    if (vecIndex) {
-      vecIndex.deserialize(payload);
+  const merged = mergeTimeOrderedSegments(contents, {
+    indexName: input.indexName,
+    partitionId: input.partitionId,
+    totalPartitions: 1,
+    language: input.language.name
+  });
+  const liveDocIds = new Set(Object.keys(merged.documents));
+  const retainedTombstones = collectRetainedTombstones(contents, liveDocIds);
+  const payload = reserialize(input, merged);
+  const id = input.nextSegmentId;
+  const key = segmentKey(input.indexName, input.partitionId, id);
+  await persistSegmentFile(input.directory, key, payload, retainedTombstones);
+  const mergedRef = {
+    id,
+    key,
+    docCount: liveDocIds.size,
+    tombstoneCount: retainedTombstones.length
+  };
+  const nextSegments = [...segments.slice(0, start), mergedRef, ...segments.slice(start + windowLength)];
+  return { segments: nextSegments, nextSegmentId: id + 1 };
+}
+function chooseCheapestWindow(segments, windowLength) {
+  let prefix = 0;
+  for (let i = 0; i < windowLength; i += 1) {
+    prefix += segments[i].docCount;
+  }
+  let bestStart = 0;
+  let bestCost = prefix;
+  for (let start = 1; start + windowLength <= segments.length; start += 1) {
+    prefix += segments[start + windowLength - 1].docCount - segments[start - 1].docCount;
+    if (prefix < bestCost) {
+      bestCost = prefix;
+      bestStart = start;
     }
   }
-  return bundle.checkpoint;
+  return bestStart;
 }
-async function loadSnapshot(directory, indexName, deps) {
-  const manifest = await readSegmentManifest(directory, indexName);
-  if (manifest !== null) {
-    return loadSegmentedSnapshot(directory, indexName, manifest, deps);
+function collectRetainedTombstones(contents, liveDocIds) {
+  const retained = /* @__PURE__ */ new Set();
+  for (const segment of contents) {
+    for (const docId of segment.tombstones) {
+      if (!liveDocIds.has(docId)) {
+        retained.add(docId);
+      }
+    }
   }
-  const bytes = await directory.read(`${indexName}/snapshot`);
-  if (bytes === null) {
+  return [...retained];
+}
+function reserialize(input, merged) {
+  const router = createPartitionRouter();
+  const vectorSink = /* @__PURE__ */ new Map();
+  const manager = createPartitionManager(input.indexName, input.config, input.language, router, 1, vectorSink);
+  manager.deserializePartition(0, merged);
+  return manager.serializePartitionToBytes(0);
+}
+async function writePartitionVectors(input) {
+  if (input.vectorFields.size === 0) {
     return [];
   }
-  return loadSnapshotBundleBytes(bytes, deps);
-}
-function segmentStartSeqNo(key, prefix) {
-  if (!key.startsWith(prefix)) {
-    return null;
+  if (input.entries.length === 0) {
+    return input.priorVectors;
   }
-  const tail = key.slice(prefix.length);
-  if (!/^\d{16}$/.test(tail)) {
-    return null;
+  const priorByField = /* @__PURE__ */ new Map();
+  for (const ref of input.priorVectors) {
+    priorByField.set(ref.fieldPath, ref);
   }
-  const value = Number.parseInt(tail, 10);
-  return Number.isSafeInteger(value) ? value : null;
-}
-async function readWalSegments(directory, indexName, partitionId) {
-  const prefix = `${indexName}/wal/${partitionId}/`;
-  const markerBytes = await directory.read(`${prefix}commit`);
-  const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
-  if (marker === null) {
-    return null;
+  const vectorIndexes = /* @__PURE__ */ new Map();
+  for (const [fieldPath, dimension] of input.vectorFields) {
+    vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dimension, input.config.vectorPromotion));
   }
-  const segments = await collectSegments(directory, prefix);
-  const durableEntries = [];
-  let highestReadFromWal = 0;
-  let activeTail = null;
-  for (const { key, startSeqNo } of segments) {
-    if (startSeqNo > marker.state.activeSegmentSeqNo) {
-      continue;
-    }
-    const bytes = await directory.read(key);
-    if (bytes === null) {
+  const router = createPartitionRouter();
+  const manager = createPartitionManager(input.indexName, input.config, input.language, router, 1, vectorIndexes);
+  for (const ref of input.priorVectors) {
+    const vecIndex = vectorIndexes.get(ref.fieldPath);
+    if (vecIndex === void 0) {
       continue;
     }
-    if (startSeqNo < marker.state.activeSegmentSeqNo) {
-      const header = checkSegmentHeader(bytes);
-      if (!header.ok) {
-        throw new NarsilError(
-          ErrorCodes.PERSISTENCE_WAL_CORRUPT,
-          `Sealed WAL segment header invalid: ${header.reason}`,
-          {
-            key,
-            reason: header.reason
-          }
-        );
-      }
-      for (const entry of readDurableRegion(bytes, bytes.length)) {
-        durableEntries.push(entry);
-        if (entry.seqNo > highestReadFromWal) {
-          highestReadFromWal = entry.seqNo;
-        }
-      }
-      continue;
+    const bytes = await input.directory.read(ref.key);
+    if (bytes !== null) {
+      const { payloadBytes } = await unpackEnvelopeBytes(bytes);
+      vecIndex.deserialize(decode(payloadBytes));
     }
-    for (const entry of readDurableRegion(bytes, marker.state.durableByteLength)) {
-      durableEntries.push(entry);
-      if (entry.seqNo > highestReadFromWal) {
-        highestReadFromWal = entry.seqNo;
-      }
+  }
+  for (const entry of input.entries) {
+    if (entry.operation === "DELETE") {
+      applyDeleteEntry(entry, manager, vectorIndexes);
+    } else {
+      applyIndexEntry(entry, manager, input.vectorFieldPaths, vectorIndexes);
     }
-    const tail = readTailBeyondFrontier(
-      bytes,
-      marker.state.durableByteLength,
-      Math.max(highestReadFromWal, marker.state.highestDurableSeqNo)
+  }
+  const result = [];
+  for (const [fieldPath, vecIndex] of vectorIndexes) {
+    const generation = (priorByField.get(fieldPath)?.generation ?? 0) + 1;
+    const key = vectorSegmentKey(input.indexName, input.partitionId, fieldPath, generation);
+    const parts = await packSnapshotEnvelopePartsRetrying(() => encode(vecIndex.serialize()));
+    await input.directory.atomicWrite(key, [parts.header, parts.payload]);
+    result.push({ fieldPath, generation, key });
+  }
+  return result;
+}
+
+// src/persistence/durability/segment/write.ts
+async function writeSegmentedCheckpoint(input) {
+  const { directory, metadata } = input;
+  const indexName = metadata.indexName;
+  const config = reconstructSchemaFromMetadata(metadata);
+  const language = getLanguage(config.language ?? "english");
+  const vectorFields = extractVectorFieldsFromSchema(config.schema);
+  const compactionThreshold = resolveCompactionThreshold(input.compactionThreshold);
+  const context = {
+    directory,
+    indexName,
+    config,
+    language,
+    vectorFields,
+    vectorFieldPaths: new Set(vectorFields.keys()),
+    compactionThreshold
+  };
+  const priorManifest = await readSegmentManifest(directory, indexName);
+  const checkpointByPartition = /* @__PURE__ */ new Map();
+  for (const target of input.targets) {
+    checkpointByPartition.set(target.partitionId, {
+      partitionId: target.partitionId,
+      lastSeqNo: target.lastSeqNo,
+      primaryTerm: target.primaryTerm
+    });
+  }
+  const partitions = [];
+  for (const target of input.targets) {
+    const priorPartition = priorManifest?.partitions.find((p) => p.partitionId === target.partitionId);
+    const priorSeqNo = snapshotCheckpointFor(priorManifest?.checkpoint ?? [], target.partitionId);
+    const entries = await collectWalEntriesInRange(
+      directory,
+      indexName,
+      target.partitionId,
+      priorSeqNo,
+      target.lastSeqNo
+    );
+    partitions.push(
+      input.wholePartitionPayload === void 0 ? await writePartition(context, target.partitionId, priorPartition, entries) : await writeWholePartition(
+        context,
+        target.partitionId,
+        priorPartition,
+        input.wholePartitionPayload(target.partitionId),
+        entries
+      )
     );
-    activeTail = { key, entries: tail.entries, cleanEnd: tail.cleanEnd, segmentLength: bytes.length };
   }
-  return { marker, segments, durableEntries, activeTail, highestReadFromWal };
+  carryForwardUncheckpointedPartitions(priorManifest, input.targets, partitions, checkpointByPartition);
+  const manifest = {
+    schema: metadata.schema,
+    language: metadata.language,
+    checkpoint: [...checkpointByPartition.values()],
+    partitions
+  };
+  const parts = await encodeSegmentManifest(manifest);
+  await directory.atomicWrite(manifestKey(indexName), [parts.header, parts.payload]);
+  await collectGarbage(directory, indexName, priorManifest, manifest);
 }
-async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
-  const read = await readWalSegments(directory, indexName, partitionId);
-  if (read === null) {
-    return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
+async function writePartition(context, partitionId, priorPartition, entries) {
+  let segments = priorPartition ? [...priorPartition.segments] : [];
+  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
+  const built = buildSegmentFromEntries({
+    indexName: context.indexName,
+    config: context.config,
+    language: context.language,
+    vectorFieldPaths: context.vectorFieldPaths,
+    entries
+  });
+  if (built !== null) {
+    const id = nextSegmentId;
+    const key = segmentKey(context.indexName, partitionId, id);
+    await persistSegmentFile(context.directory, key, built.payload, built.tombstones);
+    segments.push({ id, key, docCount: built.docCount, tombstoneCount: built.tombstones.length });
+    nextSegmentId = id + 1;
   }
-  const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
-  if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
+  if (segments.length > MAX_SEGMENTS_PER_PARTITION) {
     throw new NarsilError(
-      ErrorCodes.PERSISTENCE_WAL_CORRUPT,
-      "A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
-      {
-        indexName,
-        partitionId,
-        highestRead: highestReadFromWal,
-        highestDurable: marker.state.highestDurableSeqNo
-      }
+      ErrorCodes.PERSISTENCE_SAVE_FAILED,
+      `Partition ${partitionId} would exceed the maximum of ${MAX_SEGMENTS_PER_PARTITION} segments`,
+      { partitionId, segmentCount: segments.length, maximum: MAX_SEGMENTS_PER_PARTITION }
     );
   }
-  let highestSeqNo = fromSeqNoExclusive;
-  let highestPrimaryTerm = 0;
-  const replay = (entries) => {
-    for (const entry of entries) {
-      if (entry.seqNo <= highestSeqNo) {
-        continue;
-      }
-      applyEntry(entry, deps);
-      highestSeqNo = entry.seqNo;
-      if (entry.primaryTerm > highestPrimaryTerm) {
-        highestPrimaryTerm = entry.primaryTerm;
-      }
-    }
+  const compacted = await compactPartitionSegments({
+    directory: context.directory,
+    indexName: context.indexName,
+    partitionId,
+    config: context.config,
+    language: context.language,
+    segments,
+    nextSegmentId,
+    compactionThreshold: context.compactionThreshold
+  });
+  segments = compacted.segments;
+  nextSegmentId = compacted.nextSegmentId;
+  const vectors = await writePartitionVectors({
+    directory: context.directory,
+    indexName: context.indexName,
+    partitionId,
+    config: context.config,
+    language: context.language,
+    vectorFields: context.vectorFields,
+    vectorFieldPaths: context.vectorFieldPaths,
+    entries,
+    priorVectors: priorPartition?.vectors ?? []
+  });
+  return { partitionId, nextSegmentId, segments, vectors };
+}
+async function writeWholePartition(context, partitionId, priorPartition, whole, entries) {
+  const id = priorPartition?.nextSegmentId ?? 0;
+  const key = segmentKey(context.indexName, partitionId, id);
+  await persistSegmentFile(context.directory, key, whole.payload, []);
+  const vectors = await writePartitionVectors({
+    directory: context.directory,
+    indexName: context.indexName,
+    partitionId,
+    config: context.config,
+    language: context.language,
+    vectorFields: context.vectorFields,
+    vectorFieldPaths: context.vectorFieldPaths,
+    entries,
+    priorVectors: priorPartition?.vectors ?? []
+  });
+  return {
+    partitionId,
+    nextSegmentId: id + 1,
+    segments: [{ id, key, docCount: whole.docCount, tombstoneCount: 0 }],
+    vectors
   };
-  replay(durableEntries);
-  if (activeTail !== null) {
-    replay(activeTail.entries);
-  }
-  await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
-  if (activeTail !== null && activeTail.cleanEnd < activeTail.segmentLength) {
-    await truncateSegmentTail(directory, activeTail.key, activeTail.cleanEnd);
-  }
-  return { highestSeqNo, highestPrimaryTerm };
 }
-async function collectSegments(directory, prefix) {
-  const keys = await directory.list(prefix);
-  const refs = [];
-  for (const key of keys) {
-    const startSeqNo = segmentStartSeqNo(key, prefix);
-    if (startSeqNo !== null) {
-      refs.push({ key, startSeqNo });
-    }
+function carryForwardUncheckpointedPartitions(priorManifest, targets, partitions, checkpointByPartition) {
+  if (priorManifest === null) {
+    return;
   }
-  refs.sort((a, b) => a.startSeqNo - b.startSeqNo);
-  return refs;
-}
-async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
-  for (const { key, startSeqNo } of segments) {
-    if (startSeqNo > activeSegmentSeqNo) {
-      await directory.remove(key);
+  const covered = new Set(targets.map((t) => t.partitionId));
+  for (const priorPartition of priorManifest.partitions) {
+    if (covered.has(priorPartition.partitionId)) {
+      continue;
     }
-  }
-}
-async function truncateSegmentTail(directory, key, durableByteLength) {
-  const cleanLength = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
-  const handle = await directory.appendHandle(key);
-  try {
-    const currentSize = await handle.size();
-    if (cleanLength < currentSize) {
-      await handle.truncate(cleanLength);
+    partitions.push(priorPartition);
+    const priorCheckpoint = priorManifest.checkpoint.find((c) => c.partitionId === priorPartition.partitionId);
+    if (priorCheckpoint !== void 0 && !checkpointByPartition.has(priorPartition.partitionId)) {
+      checkpointByPartition.set(priorPartition.partitionId, priorCheckpoint);
     }
-  } finally {
-    await handle.close();
   }
 }
-function applyEntry(entry, deps) {
-  if (entry.operation === "DELETE") {
-    applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
-    return;
+function resolveCompactionThreshold(configured) {
+  if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
+    return DEFAULT_COMPACTION_THRESHOLD;
   }
-  applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes);
-}
-function snapshotCheckpointFor(checkpoint, partitionId) {
-  return checkpointLastSeqNo(checkpoint, partitionId);
+  return configured;
 }
-async function collectWalEntriesInRange(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive) {
-  const read = await readWalSegments(directory, indexName, partitionId);
-  if (read === null) {
-    return [];
+async function collectGarbage(directory, indexName, priorManifest, manifest) {
+  const referenced = manifestReferencedKeys(manifest);
+  const partitionIds = /* @__PURE__ */ new Set();
+  for (const partition of manifest.partitions) {
+    partitionIds.add(partition.partitionId);
   }
-  if (Math.max(fromSeqNoExclusive, read.highestReadFromWal) < read.marker.state.highestDurableSeqNo) {
-    throw new NarsilError(
-      ErrorCodes.PERSISTENCE_WAL_CORRUPT,
-      "A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
-      {
-        indexName,
-        partitionId,
-        highestRead: read.highestReadFromWal,
-        highestDurable: read.marker.state.highestDurableSeqNo
-      }
-    );
+  if (priorManifest !== null) {
+    for (const partition of priorManifest.partitions) {
+      partitionIds.add(partition.partitionId);
+    }
   }
-  const entries = [];
-  for (const entry of read.durableEntries) {
-    if (entry.seqNo <= fromSeqNoExclusive || entry.seqNo > upToSeqNoInclusive) {
-      continue;
+  for (const partitionId of partitionIds) {
+    const prefix = segmentPrefix(indexName, partitionId);
+    for (const key of await directory.list(prefix)) {
+      if (!referenced.has(key)) {
+        await directory.remove(key);
+      }
     }
-    entries.push(entry);
   }
-  return entries;
+  await directory.remove(legacySnapshotKey(indexName));
+}
+
+// src/persistence/durability/checkpoint-write.ts
+async function writeIndexCheckpoint(input) {
+  const { directory, metadata, targets, compactionThreshold, manager } = input;
+  const offloaded = !input.fromMemory && input.canOffload && metadata.tokenizer === void 0 && metadata.stopWords === void 0 && await runCheckpointOnWorker({ root: directory.root, metadata, targets, compactionThreshold });
+  if (offloaded) {
+    return;
+  }
+  await writeSegmentedCheckpoint({
+    directory,
+    metadata,
+    targets,
+    compactionThreshold,
+    ...input.fromMemory ? {
+      wholePartitionPayload: (partitionId) => ({
+        payload: manager.serializePartitionToBytes(partitionId),
+        docCount: manager.getPartition(partitionId).count()
+      })
+    } : {}
+  });
 }
 
 // src/persistence/durability/seq-owner.ts
@@ -3232,6 +3625,7 @@ function createWalWriter(directory, config) {
 var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
 function createDurabilityManager(config, hooks, directoryOverride) {
   const directory = createDurableDirectory(config.directory ?? "");
+  const canOffloadCheckpoint = directoryOverride === void 0;
   const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
   const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
   const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
@@ -3258,14 +3652,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
     }
     hooks.onFatalError(error);
   }
-  function toError(err) {
+  function toError3(err) {
     return err instanceof Error ? err : new Error(String(err));
   }
   async function closeWriterReportingFailure(closePromise) {
     try {
       await closePromise;
     } catch (err) {
-      markFatal(toError(err));
+      markFatal(toError3(err));
     }
   }
   function getOrCreateIndexState(indexName) {
@@ -3325,7 +3719,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
         try {
           await partition.walWriter.commit();
         } catch (err) {
-          partition.failed = toError(err);
+          partition.failed = toError3(err);
           markFatal(partition.failed);
         }
       }
@@ -3339,7 +3733,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
       try {
         await checkpointIndex(indexName);
       } catch (err) {
-        markFatal(toError(err));
+        markFatal(toError3(err));
       }
     }
   }
@@ -3354,21 +3748,24 @@ function createDurabilityManager(config, hooks, directoryOverride) {
       document: record.document
     });
   }
-  async function checkpointIndex(indexName) {
+  async function checkpointIndex(indexName, fromMemory = false) {
     const indexState = indexes.get(indexName);
     if (indexState === void 0) {
       return;
     }
     if (indexState.checkpointInFlight !== null) {
-      return indexState.checkpointInFlight;
+      await indexState.checkpointInFlight;
+      if (!fromMemory) {
+        return;
+      }
     }
-    const run = performCheckpoint(indexName, indexState).finally(() => {
+    const run = performCheckpoint(indexName, indexState, fromMemory).finally(() => {
       indexState.checkpointInFlight = null;
     });
     indexState.checkpointInFlight = run;
     return run;
   }
-  async function performCheckpoint(indexName, indexState) {
+  async function performCheckpoint(indexName, indexState, fromMemory = false) {
     const manager = hooks.getManager(indexName);
     if (manager === void 0) {
       return;
@@ -3389,22 +3786,23 @@ function createDurabilityManager(config, hooks, directoryOverride) {
         try {
           await partition.walWriter.commit();
         } catch (err) {
-          partition.failed = toError(err);
+          partition.failed = toError3(err);
           markFatal(partition.failed);
           throw partition.failed;
         }
       }
     }
-    const offloaded = await runCheckpointOnWorker({
-      root: directory.root,
+    await writeIndexCheckpoint({
+      directory,
       metadata,
       targets,
-      compactionThreshold
+      compactionThreshold,
+      manager,
+      canOffload: canOffloadCheckpoint,
+      fromMemory
     });
-    if (!offloaded) {
-      await writeSegmentedCheckpoint({ directory, metadata, targets, compactionThreshold });
-    }
     await truncateCoveredSegments(directory, indexName, targets);
+    await reclaimWalBeyondCount(directory, indexName, manager.partitionCount, indexState.partitions, markFatal);
     indexState.mutationsSinceCheckpoint = 0;
   }
   async function recoverIndex(indexName) {
@@ -3464,7 +3862,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
           const entry = buildMutationEntry(record, allocatedSeqNo);
           await partition.walWriter.append(entry);
         } catch (err) {
-          partition.failed = toError(err);
+          partition.failed = toError3(err);
           markFatal(partition.failed);
           throw partition.failed;
         }
@@ -3476,7 +3874,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
         try {
           await partition.walWriter.commit();
         } catch (err) {
-          partition.failed = toError(err);
+          partition.failed = toError3(err);
           markFatal(partition.failed);
           throw partition.failed;
         }
@@ -3486,7 +3884,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
       startAsyncFlushTimer();
       if (indexState.mutationsSinceCheckpoint >= checkpointMutationThreshold) {
         void checkpointIndex(record.indexName).catch((err) => {
-          markFatal(toError(err));
+          markFatal(toError3(err));
         });
       }
       return allocatedSeqNo;
@@ -3503,6 +3901,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
     checkpoint(indexName) {
       return checkpointIndex(indexName);
     },
+    checkpointFromMemory(indexName) {
+      return checkpointIndex(indexName, true);
+    },
     async checkpointAll() {
       for (const indexName of [...indexes.keys()]) {
         await checkpointIndex(indexName);
@@ -3548,7 +3949,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
 function metadataKey(indexName) {
   return `${indexName}/meta`;
 }
-function createSnapshotOnlyManager(adapter, config, hooks) {
+function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
   const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
   const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
   const indexes = /* @__PURE__ */ new Map();
@@ -3597,6 +3998,10 @@ function createSnapshotOnlyManager(adapter, config, hooks) {
     const { parts } = await buildSnapshotBundleBytes({
       schema: metadata.schema,
       language: metadata.language,
+      ...metadata.analysisRevision !== void 0 ? { analysisRevision: metadata.analysisRevision } : {},
+      ...metadata.tokenizer !== void 0 ? { tokenizer: metadata.tokenizer } : {},
+      ...metadata.stopWords !== void 0 ? { stopWords: metadata.stopWords } : {},
+      ...metadata.stopWordList !== void 0 ? { stopWordList: metadata.stopWordList } : {},
       manager,
       vectorIndexes: hooks.getVectorIndexes(indexName),
       seqNoByPartition,
@@ -3604,6 +4009,13 @@ function createSnapshotOnlyManager(adapter, config, hooks) {
     });
     await adapter.save(snapshotStorageKey(indexName), concatEnvelopeParts(parts));
     indexState.mutationsSinceCheckpoint = 0;
+    if (publisher !== void 0) {
+      const partitionIds = [];
+      for (let i = 0; i < manager.partitionCount; i += 1) {
+        partitionIds.push(i);
+      }
+      await publisher.publishPartitions(indexName, partitionIds);
+    }
   }
   async function checkpointIndex(indexName) {
     const indexState = indexes.get(indexName);
@@ -3690,6 +4102,12 @@ function createSnapshotOnlyManager(adapter, config, hooks) {
         await adapter.delete(key);
       }
     },
+    async reloadIndex(indexName) {
+      const indexState = getOrCreateIndexState(indexName);
+      const reload = indexState.applyChain.then(() => recoverIndex(indexName));
+      indexState.applyChain = reload.catch(() => void 0);
+      await reload;
+    },
     async shutdown() {
       shuttingDown = true;
       if (checkpointTimer !== null) {
@@ -3728,6 +4146,36 @@ function buildMetadata(indexName, hooks) {
   if (config.surfaceForms) {
     metadata.surfaceForms = true;
   }
+  if (config.analysisRevision !== void 0) {
+    metadata.analysisRevision = config.analysisRevision;
+  }
+  if (config.tokenizer !== void 0) {
+    metadata.tokenizer = config.tokenizer;
+  }
+  if (config.stopWords !== void 0) {
+    metadata.stopWords = config.stopWords;
+  }
+  if (config.stopWordList !== void 0) {
+    metadata.stopWordList = config.stopWordList;
+  }
+  if (config.partitionLimits !== void 0) {
+    metadata.partitionLimits = config.partitionLimits;
+  }
+  if (config.defaultScoring !== void 0) {
+    metadata.defaultScoring = config.defaultScoring;
+  }
+  if (config.trackPositions !== void 0) {
+    metadata.trackPositions = config.trackPositions;
+  }
+  if (config.strict !== void 0) {
+    metadata.strict = config.strict;
+  }
+  if (config.required !== void 0) {
+    metadata.required = config.required;
+  }
+  if (config.vectorPromotion !== void 0) {
+    metadata.vectorPromotion = config.vectorPromotion;
+  }
   return metadata;
 }
 function createDurabilityIntegration(tier, hooks) {
@@ -3739,7 +4187,7 @@ function createDurabilityIntegration(tier, hooks) {
     createIndexFromMetadata: hooks.createIndexFromMetadata,
     onFatalError: hooks.onFatalError
   };
-  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks);
+  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
   function partitionFor(indexName, docId) {
     const partitionManager = hooks.getManager(indexName);
     const partitionCount = partitionManager?.partitionCount ?? 1;
@@ -3758,12 +4206,168 @@ function createDurabilityIntegration(tier, hooks) {
     return { indexName, partitionId, seqNo };
   }
   return {
-    manager,
-    recordInsertOrUpdate(indexName, docId, document, apply) {
-      return recordMutation(indexName, docId, "INDEX", encode(document), apply);
+    manager,
+    recordInsertOrUpdate(indexName, docId, document, apply) {
+      return recordMutation(indexName, docId, "INDEX", encode(document), apply);
+    },
+    recordRemove(indexName, docId, apply) {
+      return recordMutation(indexName, docId, "DELETE", null, apply);
+    }
+  };
+}
+
+// src/engine/invalidation.ts
+var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
+var FOREIGN_STATS_TTL_MS = 6e4;
+function createInvalidationFromConfig(config, tierKind, deps) {
+  if (config?.invalidation === void 0) {
+    return null;
+  }
+  if (tierKind !== "snapshot") {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      "Invalidation requires a shared persistence adapter. The write-ahead log tier owns its directory exclusively, so configure a non-filesystem persistence adapter to share data between instances"
+    );
+  }
+  return createInvalidationIntegration({
+    adapter: config.invalidation,
+    instanceId: generateId(),
+    ...deps
+  });
+}
+function createInvalidationIntegration(deps) {
+  const { adapter, instanceId, reloadIndex, getManager, listBroadcastIndexNames, onError } = deps;
+  const foreignStats = /* @__PURE__ */ new Map();
+  const reloadChains = /* @__PURE__ */ new Map();
+  let broadcastTimer = null;
+  let stopped = false;
+  function toError3(err) {
+    return err instanceof Error ? err : new Error(String(err));
+  }
+  function pruneStaleForeignStats(entries) {
+    const cutoff = Date.now() - FOREIGN_STATS_TTL_MS;
+    for (const [sourceInstanceId, entry] of entries) {
+      if (entry.receivedAt < cutoff) {
+        entries.delete(sourceInstanceId);
+      }
+    }
+  }
+  function scheduleReload(indexName) {
+    const chain = reloadChains.get(indexName) ?? Promise.resolve();
+    const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(toError3(err)));
+    reloadChains.set(indexName, next);
+  }
+  function storeForeignStats(indexName, sourceInstanceId, stats) {
+    let entries = foreignStats.get(indexName);
+    if (entries === void 0) {
+      entries = /* @__PURE__ */ new Map();
+      foreignStats.set(indexName, entries);
+    }
+    entries.set(sourceInstanceId, { stats, receivedAt: Date.now() });
+    pruneStaleForeignStats(entries);
+  }
+  function handleEvent(event) {
+    try {
+      if (event.type === "partition") {
+        if (event.sourceInstanceId === instanceId) {
+          return;
+        }
+        scheduleReload(event.indexName);
+        return;
+      }
+      if (event.instanceId === instanceId) {
+        return;
+      }
+      storeForeignStats(event.indexName, event.instanceId, event.stats);
+    } catch (err) {
+      onError(toError3(err));
+    }
+  }
+  async function publishOwnStatistics() {
+    for (const indexName of listBroadcastIndexNames()) {
+      const manager = getManager(indexName);
+      if (manager === void 0) {
+        continue;
+      }
+      const aggregate = manager.getAggregateStats();
+      try {
+        await adapter.publish({
+          type: "statistics",
+          indexName,
+          instanceId,
+          stats: {
+            totalDocs: aggregate.totalDocuments,
+            docFrequencies: aggregate.docFrequencies,
+            totalFieldLengths: aggregate.totalFieldLengths
+          }
+        });
+      } catch (err) {
+        onError(toError3(err));
+      }
+    }
+  }
+  return {
+    instanceId,
+    async publishPartitions(indexName, partitions) {
+      if (stopped || partitions.length === 0) {
+        return;
+      }
+      try {
+        await adapter.publish({
+          type: "partition",
+          indexName,
+          partitions,
+          timestamp: Date.now(),
+          sourceInstanceId: instanceId
+        });
+      } catch (err) {
+        onError(toError3(err));
+      }
     },
-    recordRemove(indexName, docId, apply) {
-      return recordMutation(indexName, docId, "DELETE", null, apply);
+    broadcastStats(indexName) {
+      const manager = getManager(indexName);
+      if (manager === void 0) {
+        return void 0;
+      }
+      const own = manager.getAggregateStats();
+      const entries = foreignStats.get(indexName);
+      if (entries === void 0 || entries.size === 0) {
+        return mergePartitionStats([own]);
+      }
+      pruneStaleForeignStats(entries);
+      const combined = [own];
+      for (const [, entry] of entries) {
+        combined.push({
+          totalDocuments: entry.stats.totalDocs,
+          docFrequencies: entry.stats.docFrequencies,
+          totalFieldLengths: entry.stats.totalFieldLengths
+        });
+      }
+      return mergePartitionStats(combined);
+    },
+    async start() {
+      await adapter.subscribe(handleEvent);
+      if (broadcastTimer !== null || stopped) {
+        return;
+      }
+      broadcastTimer = setInterval(() => {
+        void publishOwnStatistics();
+      }, STATISTICS_BROADCAST_INTERVAL_MS);
+      if (typeof broadcastTimer.unref === "function") {
+        broadcastTimer.unref();
+      }
+    },
+    async shutdown() {
+      stopped = true;
+      if (broadcastTimer !== null) {
+        clearInterval(broadcastTimer);
+        broadcastTimer = null;
+      }
+      try {
+        await adapter.shutdown();
+      } catch (err) {
+        onError(toError3(err));
+      }
     }
   };
 }
@@ -4016,42 +4620,174 @@ function createWorkerPool(config) {
   };
 }
 
+// src/engine/worker-resync.ts
+async function transferIndexToPool(indexName, pool, config, manager) {
+  const allExecutors = pool.getAllExecutors();
+  await Promise.allSettled(
+    allExecutors.map(
+      (workerExecutor) => workerExecutor.execute({ type: "dropIndex", indexName, requestId: `resync-drop-${indexName}` })
+    )
+  );
+  pool.addIndexToAll(indexName);
+  await Promise.all(
+    allExecutors.map(
+      (workerExecutor) => workerExecutor.execute({
+        type: "createIndex",
+        indexName,
+        config,
+        requestId: `resync-create-${indexName}`
+      })
+    )
+  );
+  for (let i = 0; i < manager.partitionCount; i++) {
+    const serialized = manager.serializePartition(i);
+    await Promise.all(
+      allExecutors.map(
+        (workerExecutor) => workerExecutor.execute({
+          type: "deserialize",
+          indexName,
+          partitionId: i,
+          data: serialized,
+          requestId: `resync-${indexName}-${i}`
+        })
+      )
+    );
+  }
+}
+
 // src/engine/orchestration.ts
+function toError2(value) {
+  return value instanceof Error ? value : new Error(String(value));
+}
+function isDeterministicFailure(error) {
+  return error instanceof NarsilError && error.code === ErrorCodes.CONFIG_INVALID;
+}
+function alreadyPresentOnWorker(reason) {
+  return reason instanceof NarsilError && reason.code === ErrorCodes.DOC_ALREADY_EXISTS;
+}
+function workerIneligibility(indexName, config, bootstrapModule) {
+  try {
+    assertConfigReachesWorker(indexName, config, bootstrapModule);
+    return null;
+  } catch (err) {
+    if (err instanceof NarsilError && err.code === ErrorCodes.CONFIG_INVALID) {
+      return err;
+    }
+    throw err;
+  }
+}
+function assertConfigReachesWorker(indexName, config, bootstrapModule) {
+  if (config.tokenizer !== void 0 && typeof config.tokenizer !== "string") {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      `Index "${indexName}" holds a tokenizer instance, and no worker thread can receive one. Register the tokenizer with registerTokenizer and name it in the index config`,
+      { indexName }
+    );
+  }
+  if (typeof config.stopWords === "function") {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      `Index "${indexName}" holds a stop word function, and no worker thread can receive one. Register the function with registerStopWords and name it in the index config`,
+      { indexName }
+    );
+  }
+  const language = config.language ?? "english";
+  if (language !== "english" && bootstrapModule === void 0) {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      `Index "${indexName}" uses language "${language}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
+      { indexName, language }
+    );
+  }
+}
 function createWorkerOrchestrator(config, executor, promoter, indexRegistry, callbacks) {
   let workerPool = null;
   let promotionInProgress = false;
+  let promotionBlocked = false;
+  let promotionRun = null;
   const promotionBuffer = [];
+  const awaitingBufferedWrites = /* @__PURE__ */ new Set();
+  const reportedIneligible = /* @__PURE__ */ new Set();
+  const promotedIndexes = /* @__PURE__ */ new Set();
   const workersEnabled = config?.workers?.enabled === true;
-  async function checkPromotion() {
-    if (!workersEnabled || promotionInProgress || workerPool) return;
-    const indexMap = /* @__PURE__ */ new Map();
-    for (const [name] of indexRegistry) {
+  const bootstrapModule = config?.workers?.bootstrapModule;
+  function reportIneligible(indexName, error) {
+    if (reportedIneligible.has(indexName)) return;
+    reportedIneligible.add(indexName);
+    callbacks?.onPromotionFailure?.("index-excluded", error, false);
+  }
+  function collectEligibleIndexes() {
+    const eligible = /* @__PURE__ */ new Map();
+    for (const [name, entry] of indexRegistry) {
+      const ineligibility = workerIneligibility(name, entry.config, bootstrapModule);
+      if (ineligibility) {
+        reportIneligible(name, ineligibility);
+        continue;
+      }
       const mgr = executor.getManager(name);
-      indexMap.set(name, { documentCount: mgr?.countDocuments() ?? 0 });
+      eligible.set(name, { documentCount: mgr?.countDocuments() ?? 0 });
     }
+    return eligible;
+  }
+  async function checkPromotion() {
+    if (!workersEnabled || promotionInProgress || promotionBlocked || workerPool) return;
+    if (callbacks?.shouldDeferPromotion?.()) return;
+    const indexMap = collectEligibleIndexes();
+    if (indexMap.size === 0) return;
     const result = promoter.check(indexMap);
     if (result.shouldPromote) {
       promotionInProgress = true;
       setTimeout(() => {
-        runPromotion(result.reason).catch((err) => {
-          console.warn("Worker promotion failed:", err);
+        const run = runPromotion(result.reason).catch((err) => {
+          const error = toError2(err);
+          promotionBlocked = isDeterministicFailure(error);
           promotionInProgress = false;
+          callbacks?.onPromotionFailure?.(result.reason, error, !promotionBlocked);
+        });
+        promotionRun = run.then(() => {
+          promotionRun = null;
         });
       }, 0);
     }
   }
   async function runPromotion(reason) {
     try {
+      const promotable = [];
+      for (const [name, entry] of indexRegistry) {
+        const ineligibility = workerIneligibility(name, entry.config, bootstrapModule);
+        if (ineligibility) {
+          reportIneligible(name, ineligibility);
+          continue;
+        }
+        promotable.push(name);
+      }
+      if (promotable.length === 0) {
+        promotionBuffer.length = 0;
+        return;
+      }
       const factory = await createWorkerFactory();
       const pool = createWorkerPool({
         count: config?.workers?.count,
         workerFactory: factory
       });
-      for (const [name] of indexRegistry) {
+      for (const name of promotable) {
         pool.addIndexToAll(name);
       }
       const allExecutors = pool.getAllExecutors();
-      for (const [name, entry] of indexRegistry) {
+      if (bootstrapModule !== void 0) {
+        await Promise.all(
+          allExecutors.map(
+            (workerExecutor) => workerExecutor.execute({
+              type: "bootstrap",
+              moduleUrl: bootstrapModule,
+              requestId: "promote-bootstrap"
+            })
+          )
+        );
+      }
+      for (const name of promotable) {
+        const entry = indexRegistry.get(name);
+        if (!entry) continue;
         await Promise.all(
           allExecutors.map(
             (workerExecutor) => workerExecutor.execute({
@@ -4081,38 +4817,64 @@ function createWorkerOrchestrator(config, executor, promoter, indexRegistry, cal
         }
       }
       workerPool = pool;
-      promoter.markPromoted();
-      if (promotionBuffer.length > 0) {
-        const buffered = [...promotionBuffer];
-        promotionBuffer.length = 0;
-        for (const action of buffered) {
-          await replicateToWorkers(action);
-        }
+      for (const name of promotable) {
+        promotedIndexes.add(name);
+        awaitingBufferedWrites.add(name);
       }
+      promoter.markPromoted();
+      await drainPromotionBuffer();
+      awaitingBufferedWrites.clear();
       callbacks?.onPromotion?.(pool.workerCount, reason);
     } catch (err) {
       promotionBuffer.length = 0;
       throw err;
     } finally {
+      awaitingBufferedWrites.clear();
       promotionInProgress = false;
     }
   }
+  async function drainPromotionBuffer() {
+    while (promotionBuffer.length > 0) {
+      const action = promotionBuffer.shift();
+      if (action === void 0) return;
+      await dispatchToWorkers(action);
+    }
+  }
   async function replicateToWorkers(action) {
     if (promotionInProgress) {
       promotionBuffer.push(action);
       return;
     }
+    await dispatchToWorkers(action);
+  }
+  async function dispatchToWorkers(action) {
     if (!workerPool) return;
+    if (action.type === "createIndex") {
+      const ineligibility = workerIneligibility(action.indexName, action.config, bootstrapModule);
+      if (ineligibility) {
+        reportIneligible(action.indexName, ineligibility);
+        return;
+      }
+      workerPool.addIndexToAll(action.indexName);
+      promotedIndexes.add(action.indexName);
+    } else if ("indexName" in action && !promotedIndexes.has(action.indexName)) {
+      return;
+    }
     const allExecutors = workerPool.getAllExecutors();
     const results = await Promise.allSettled(allExecutors.map((workerExecutor) => workerExecutor.execute(action)));
     for (const result of results) {
       if (result.status === "rejected") {
+        if (action.type === "insert" && alreadyPresentOnWorker(result.reason)) {
+          continue;
+        }
         console.warn("Worker replication failed:", result.reason);
       }
     }
   }
-  async function searchViaWorker(indexName, params) {
+  async function searchViaWorker(indexName, params, globalStats) {
     if (!workerPool) return null;
+    if (!promotedIndexes.has(indexName)) return null;
+    if (awaitingBufferedWrites.has(indexName)) return null;
     const manager = executor.getManager(indexName);
     if (!manager) return null;
     const allExecutors = workerPool.getAllExecutors();
@@ -4125,7 +4887,8 @@ function createWorkerOrchestrator(config, executor, promoter, indexRegistry, cal
           type: "query",
           indexName,
           params,
-          requestId: createRequestId()
+          requestId: createRequestId(),
+          ...globalStats !== void 0 ? { globalStats } : {}
         });
       } catch (err) {
         console.warn("Worker search failed, falling back to local:", err);
@@ -4144,7 +4907,8 @@ function createWorkerOrchestrator(config, executor, promoter, indexRegistry, cal
             indexName,
             params,
             requestId: createRequestId(),
-            partitionIds: workerAssignments[idx]
+            partitionIds: workerAssignments[idx],
+            ...globalStats !== void 0 ? { globalStats } : {}
           })
         )
       );
@@ -4163,6 +4927,24 @@ function createWorkerOrchestrator(config, executor, promoter, indexRegistry, cal
   function isPromoted() {
     return workerPool !== null;
   }
+  function desyncIndex(indexName) {
+    return promotedIndexes.delete(indexName);
+  }
+  async function resyncIndex(indexName, wasPromoted) {
+    if (promotionRun) {
+      await promotionRun.catch(() => void 0);
+    }
+    if (!workerPool) return;
+    if (!wasPromoted && !promotedIndexes.has(indexName)) return;
+    const entry = indexRegistry.get(indexName);
+    if (!entry) return;
+    if (workerIneligibility(indexName, entry.config, bootstrapModule)) return;
+    const manager = executor.getManager(indexName);
+    if (!manager) return;
+    promotedIndexes.delete(indexName);
+    await transferIndexToPool(indexName, workerPool, entry.config, manager);
+    promotedIndexes.add(indexName);
+  }
   async function getWorkerMemoryStats() {
     if (!workerPool) return [];
     const reports = await workerPool.getMemoryStats();
@@ -4177,59 +4959,65 @@ function createWorkerOrchestrator(config, executor, promoter, indexRegistry, cal
     if (workerPool) {
       await workerPool.shutdown();
       workerPool = null;
+      promotedIndexes.clear();
     }
   }
-  return { checkPromotion, replicateToWorkers, searchViaWorker, isPromoted, getWorkerMemoryStats, shutdown };
+  return {
+    checkPromotion,
+    replicateToWorkers,
+    searchViaWorker,
+    isPromoted,
+    desyncIndex,
+    resyncIndex,
+    getWorkerMemoryStats,
+    shutdown
+  };
 }
 
-// src/engine/core.ts
-function getVectorFieldPaths(schema) {
-  return new Set(extractVectorFieldsFromSchema(schema).keys());
-}
-function filesystemBackedDirectory(config) {
-  const adapterDirectory = config.persistence?.directory;
-  if (adapterDirectory !== void 0 && adapterDirectory.trim().length > 0) {
-    return adapterDirectory;
-  }
-  return null;
-}
-function resolveDurabilityTier(config) {
-  const filesystemDirectory = filesystemBackedDirectory(config);
-  if (config.durability) {
-    if (config.durability.directory !== void 0 && config.durability.directory.trim().length > 0) {
-      return { kind: "wal", config: { ...config.durability } };
+// src/engine/watermark.ts
+function createWatermarkNotifier(deps) {
+  const latchedCapacity = /* @__PURE__ */ new Map();
+  function check(indexName) {
+    const partitionConfig = deps.getPartitionConfig(indexName);
+    const maxDocs = partitionConfig?.maxDocsPerPartition;
+    const watermark = partitionConfig?.watermark;
+    if (maxDocs === void 0 || watermark === void 0) {
+      latchedCapacity.delete(indexName);
+      return;
     }
-    if (filesystemDirectory !== null) {
-      return { kind: "wal", config: { ...config.durability, directory: filesystemDirectory } };
+    const manager = deps.getManager(indexName);
+    if (!manager) {
+      latchedCapacity.delete(indexName);
+      return;
     }
-    if (config.persistence !== void 0) {
-      throw new NarsilError(
-        ErrorCodes.CONFIG_INVALID,
-        "WAL durability requires a filesystem directory, but the configured persistence adapter is not filesystem-backed. Use a filesystem persistence adapter or set durability.directory"
-      );
+    const partitionCount = manager.partitionCount;
+    const capacity = maxDocs * partitionCount;
+    const documentCount = manager.countDocuments();
+    const threshold = watermark * capacity;
+    if (documentCount < threshold) {
+      latchedCapacity.delete(indexName);
+      return;
     }
-    throw new NarsilError(
-      ErrorCodes.CONFIG_INVALID,
-      "Durability requires a directory. Set durability.directory explicitly, or configure a filesystem persistence adapter"
-    );
-  }
-  if (config.persistence === void 0) {
-    return null;
+    if (latchedCapacity.get(indexName) === capacity) return;
+    latchedCapacity.set(indexName, capacity);
+    deps.emit({ indexName, documentCount, capacity, partitionCount });
   }
-  if (filesystemDirectory !== null) {
-    return { kind: "wal", config: { directory: filesystemDirectory } };
+  function forget(indexName) {
+    latchedCapacity.delete(indexName);
   }
-  return { kind: "snapshot", config: {}, adapter: config.persistence };
+  return { check, forget };
 }
-function createDurabilityFromConfig(config, wiring) {
-  if (config === void 0) {
-    return null;
-  }
-  const tier = resolveDurabilityTier(config);
+
+// src/engine/core.ts
+function getVectorFieldPaths(schema) {
+  return new Set(extractVectorFieldsFromSchema(schema).keys());
+}
+function createDurabilityFromTier(tier, wiring) {
   if (tier === null) {
     return null;
   }
   return createDurabilityIntegration(tier, {
+    checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
     getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.requireManager(indexName) : void 0,
     getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
     getVectorIndexes: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.requireManager(indexName).getVectorIndexes() : /* @__PURE__ */ new Map(),
@@ -4248,7 +5036,17 @@ function createDurabilityFromConfig(config, wiring) {
         k1: entry.config.bm25?.k1 ?? 1.2,
         b: entry.config.bm25?.b ?? 0.75,
         ...embedding !== void 0 ? { embedding } : {},
-        ...entry.config.surfaceForms === true ? { surfaceForms: true } : {}
+        ...entry.config.surfaceForms === true ? { surfaceForms: true } : {},
+        analysisRevision: entry.language.revision,
+        ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
+        ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
+        ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort() } : {},
+        ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
+        ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
+        ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
+        ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
+        ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
+        ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {}
       };
     },
     createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -4277,19 +5075,30 @@ function createEngineCore(config) {
   const eventHandlers = /* @__PURE__ */ new Map();
   const shutdownState = { isShutdown: false };
   const abortController = new AbortController();
+  const rebalancingIndexes = /* @__PURE__ */ new Set();
   const orchestrator = createWorkerOrchestrator(config, executor, promoter, indexRegistry, {
+    shouldDeferPromotion() {
+      return rebalancingIndexes.size > 0;
+    },
     onPromotion(workerCount, reason) {
       const handlers = eventHandlers.get("workerPromote");
       if (handlers) {
         for (const handler of handlers) handler({ workerCount, reason });
       }
+    },
+    onPromotionFailure(reason, error, retryable) {
+      const handlers = eventHandlers.get("workerPromoteFailure");
+      if (!handlers || handlers.size === 0) {
+        console.warn(`Worker promotion failed (${reason}):`, error);
+        return;
+      }
+      for (const handler of handlers) handler({ reason, error, retryable });
     }
   });
   const rebalancer = createRebalancer();
   const rebalanceRouter = createPartitionRouter();
-  const rebalancingIndexes = /* @__PURE__ */ new Set();
   const waqMap = /* @__PURE__ */ new Map();
-  const lastAppliedSeqMap = /* @__PURE__ */ new Map();
+  const rebalanceTargets = /* @__PURE__ */ new Map();
   function guardShutdown() {
     if (shutdownState.isShutdown) {
       throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "This Narsil instance has been shut down");
@@ -4346,7 +5155,18 @@ function createEngineCore(config) {
         embeddingAdapter = candidate;
       }
     }
-    executor.createIndex(metadata.indexName, indexConfig, language);
+    try {
+      executor.createIndex(metadata.indexName, indexConfig, language);
+    } catch (err) {
+      if (err instanceof NarsilError && err.code === ErrorCodes.CONFIG_INVALID) {
+        throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
+          indexName: metadata.indexName,
+          tokenizer: metadata.tokenizer,
+          stopWords: metadata.stopWords
+        });
+      }
+      throw err;
+    }
     indexRegistry.set(metadata.indexName, {
       config: indexConfig,
       language,
@@ -4354,8 +5174,18 @@ function createEngineCore(config) {
       embeddingAdapterName: adapterName,
       vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
     });
+    if (metadata.analysisRevision !== language.revision) {
+      analysisRebuild.markStale({
+        indexName: metadata.indexName,
+        language: language.name,
+        storedRevision: metadata.analysisRevision ?? null,
+        currentRevision: language.revision
+      });
+    }
   }
-  const durability = createDurabilityFromConfig(config, {
+  const durabilityTier = config !== void 0 ? resolveDurabilityTier(config) : null;
+  let invalidation = null;
+  const durability = createDurabilityFromTier(durabilityTier, {
     requireManager,
     indexRegistry,
     createIndexFromMetadata,
@@ -4364,8 +5194,53 @@ function createEngineCore(config) {
       if (handlers) {
         for (const handler of handlers) handler({ error });
       }
+    },
+    publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve()
+  });
+  invalidation = createInvalidationFromConfig(config, durabilityTier?.kind ?? null, {
+    getManager: (indexName) => executor.getManager(indexName) ?? void 0,
+    listBroadcastIndexNames: () => {
+      const names = [];
+      for (const [name, entry] of indexRegistry) {
+        if (entry.config.defaultScoring === "broadcast") {
+          names.push(name);
+        }
+      }
+      return names;
+    },
+    reloadIndex: (indexName) => durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve(),
+    onError(error) {
+      const handlers = eventHandlers.get("invalidationError");
+      if (!handlers || handlers.size === 0) {
+        console.warn("Invalidation error:", error);
+        return;
+      }
+      for (const handler of handlers) handler({ error });
+    }
+  });
+  const watermarkNotifier = createWatermarkNotifier({
+    getManager: (indexName) => executor.getManager(indexName),
+    getPartitionConfig: (indexName) => indexRegistry.get(indexName)?.config.partitions,
+    emit(payload) {
+      const handlers = eventHandlers.get("partitionWatermark");
+      if (!handlers) return;
+      for (const handler of handlers) {
+        try {
+          handler(payload);
+        } catch (err) {
+          console.warn("partitionWatermark handler error:", err instanceof Error ? err.message : String(err));
+        }
+      }
     }
   });
+  const analysisRebuild = wireAnalysisRebuild({
+    config: config?.analysis,
+    eventHandlers,
+    getManager: (indexName) => executor.getManager(indexName),
+    desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
+    resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
+    durabilityManager: durability?.manager ?? null
+  });
   const mutationCtx = {
     executor,
     pluginRegistry,
@@ -4376,15 +5251,24 @@ function createEngineCore(config) {
     guardShutdown,
     requireIndex,
     requireManager,
-    bufferIfRebalancing
+    bufferIfRebalancing,
+    isRebalancing: (indexName) => rebalancingIndexes.has(indexName),
+    pendingRebalanceWrites: (indexName) => waqMap.get(indexName)?.size ?? 0,
+    rebalanceTargetPartitionCount: (indexName) => rebalanceTargets.get(indexName),
+    bufferedDocState: (indexName, docId) => waqMap.get(indexName)?.bufferedDocState(docId),
+    checkWatermark: watermarkNotifier.check
   };
   const rebalanceCtx = {
     rebalancer,
     router: rebalanceRouter,
     waqMap,
     rebalancingIndexes,
-    lastAppliedSeqMap,
+    rebalanceTargets,
     eventHandlers,
+    pluginRegistry,
+    orchestrator,
+    durabilityManager: durability?.manager ?? null,
+    checkWatermark: watermarkNotifier.check,
     requireIndex
   };
   return {
@@ -4392,6 +5276,7 @@ function createEngineCore(config) {
     promoter,
     pluginRegistry,
     durability,
+    invalidation,
     idGenerator,
     indexRegistry,
     embeddingAdapters,
@@ -4403,11 +5288,12 @@ function createEngineCore(config) {
     rebalanceRouter,
     rebalancingIndexes,
     waqMap,
-    lastAppliedSeqMap,
     guardShutdown,
     requireIndex,
     requireManager,
     bufferIfRebalancing,
+    watermarkNotifier,
+    analysisRebuild,
     mutationCtx,
     rebalanceCtx
   };
@@ -4587,6 +5473,24 @@ function validateDocId(docId) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
   }
 }
+function validatePartitionConfig(partitions) {
+  const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
+  if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxDocsPerPartition must be a positive integer", {
+      maxDocsPerPartition
+    });
+  }
+  if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxPartitions must be a positive integer", {
+      maxPartitions
+    });
+  }
+  if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
+      watermark
+    });
+  }
+}
 function clampLimit(limit) {
   if (limit === void 0) return DEFAULT_LIMIT;
   return Math.max(0, Math.min(limit, MAX_LIMIT));
@@ -4605,6 +5509,25 @@ async function createEngineIndex(core, config, name, indexConfig) {
   }
   validateSchema(indexConfig.schema);
   validateVectorPromotion(indexConfig.vectorPromotion);
+  if (indexConfig.partitions) {
+    validatePartitionConfig(indexConfig.partitions);
+  }
+  if (core.durability) {
+    if (indexConfig.tokenizer !== void 0 && typeof indexConfig.tokenizer !== "string") {
+      throw new NarsilError(
+        ErrorCodes.CONFIG_INVALID,
+        `Index "${name}" cannot use a tokenizer instance while durability is configured, because a checkpoint persists no code. Register it with registerTokenizer and pass its name`,
+        { indexName: name }
+      );
+    }
+    if (typeof indexConfig.stopWords === "function") {
+      throw new NarsilError(
+        ErrorCodes.CONFIG_INVALID,
+        `Index "${name}" cannot use a stop word function while durability is configured, because a checkpoint persists no code. Register it with registerStopWords and pass its name`,
+        { indexName: name }
+      );
+    }
+  }
   let resolvedEmbeddingAdapter = null;
   let embeddingAdapterName = null;
   if (indexConfig.embedding) {
@@ -4649,6 +5572,8 @@ async function dropEngineIndex(core, name) {
   const entry = core.requireIndex(name);
   core.executor.dropIndex(name);
   core.indexRegistry.delete(name);
+  core.watermarkNotifier.forget(name);
+  core.analysisRebuild.clearStale(name);
   if (core.durability) {
     await core.durability.manager.removeIndex(name);
   }
@@ -4697,6 +5622,9 @@ async function shutdownEngine(core) {
   if (durability) {
     await durability.manager.shutdown();
   }
+  if (core.invalidation) {
+    await core.invalidation.shutdown();
+  }
   const adaptersToShutdown = /* @__PURE__ */ new Set();
   for (const [, entry] of indexRegistry) {
     if (entry.embeddingAdapter?.shutdown) {
@@ -4979,14 +5907,23 @@ function providedDocId(document) {
   const id = document.id;
   return typeof id === "string" && id.length > 0 ? id : void 0;
 }
+function admitInsert(ctx, indexName, manager, docId) {
+  if (!ctx.isRebalancing(indexName)) {
+    manager.assertCapacity();
+    return;
+  }
+  const bufferedState = ctx.bufferedDocState(indexName, docId);
+  const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
+  if (exists) {
+    throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
+  }
+  manager.assertCapacity(ctx.pendingRebalanceWrites(indexName), ctx.rebalanceTargetPartitionCount(indexName));
+}
 async function insertDocument(ctx, indexName, document, docId, options) {
   ctx.guardShutdown();
   const entry = ctx.requireIndex(indexName);
   const resolvedDocId = docId ?? providedDocId(document) ?? ctx.idGenerator();
   validateDocId(resolvedDocId);
-  if (ctx.bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
-    return resolvedDocId;
-  }
   await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document });
   if (entry.config.required && entry.config.required.length > 0) {
     validateRequiredFields(document, entry.config.required);
@@ -5008,6 +5945,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
     }
   }
   const insertManager = ctx.requireManager(indexName);
+  admitInsert(ctx, indexName, insertManager, resolvedDocId);
   const insertVecIndexes = insertManager.getVectorIndexes();
   const { partitionDoc, extractedVectors } = prepareDocumentVectors(
     document,
@@ -5018,7 +5956,13 @@ async function insertDocument(ctx, indexName, document, docId, options) {
     validateVectorDimensions(extractedVectors, insertVecIndexes);
   }
   let inserted = false;
+  let buffered = false;
   const applyInsert = async () => {
+    admitInsert(ctx, indexName, insertManager, resolvedDocId);
+    if (ctx.bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
+      buffered = true;
+      return;
+    }
     await ctx.executor.execute({
       type: "insert",
       indexName,
@@ -5053,6 +5997,10 @@ async function insertDocument(ctx, indexName, document, docId, options) {
   } else {
     await applyInsert();
   }
+  if (buffered) {
+    ctx.checkWatermark(indexName);
+    return resolvedDocId;
+  }
   try {
     await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
   } catch (err) {
@@ -5072,6 +6020,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
       vecIndex.scheduleBuild();
     }
   }
+  ctx.checkWatermark(indexName);
   await ctx.orchestrator.checkPromotion();
   return resolvedDocId;
 }
@@ -5090,6 +6039,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
   const batchVecIndexes = batchManager.getVectorIndexes();
   const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
   const touchedVectorFields = /* @__PURE__ */ new Set();
+  const bufferedDocIds = /* @__PURE__ */ new Set();
   for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
     if (ctx.abortController.signal.aborted) break;
     const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
@@ -5171,8 +6121,20 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
         if (extractedVectors.size > 0) {
           validateVectorDimensions(extractedVectors, batchVecIndexes);
         }
+        admitInsert(ctx, indexName, batchManager, batchDocId);
         let batchInserted = false;
+        let batchBuffered = false;
         const applyBatchInsert = async () => {
+          admitInsert(ctx, indexName, batchManager, batchDocId);
+          if (ctx.bufferIfRebalancing(indexName, {
+            action: "insert",
+            docId: batchDocId,
+            document: documents[i],
+            indexName
+          })) {
+            batchBuffered = true;
+            return;
+          }
           const result = ctx.executor.execute({
             type: "insert",
             indexName,
@@ -5210,6 +6172,12 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
         } else {
           await applyBatchInsert();
         }
+        if (batchBuffered) {
+          bufferedDocIds.add(batchDocId);
+          succeeded.push(batchDocId);
+          succeededDocs.push(documents[i]);
+          continue;
+        }
         for (const fieldPath of extractedVectors.keys()) {
           touchedVectorFields.add(fieldPath);
         }
@@ -5234,6 +6202,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
     }
   }
   for (let i = 0; i < succeeded.length; i++) {
+    if (bufferedDocIds.has(succeeded[i])) continue;
     await ctx.orchestrator.replicateToWorkers({
       type: "insert",
       indexName,
@@ -5249,6 +6218,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
       vecIndex.scheduleBuild();
     }
   }
+  ctx.checkWatermark(indexName);
   await ctx.orchestrator.checkPromotion();
   return { succeeded, failed };
 }
@@ -5258,16 +6228,26 @@ async function removeDocument(ctx, indexName, docId) {
   ctx.guardShutdown();
   ctx.requireIndex(indexName);
   validateDocId(docId);
-  if (ctx.bufferIfRebalancing(indexName, { action: "remove", docId, indexName })) {
-    return;
-  }
   await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
-  if (ctx.durability && !ctx.requireManager(indexName).has(docId)) {
+  const rebalancing = ctx.isRebalancing(indexName);
+  if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
     throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
   }
-  const restoreRef = ctx.durability ? ctx.requireManager(indexName).getRef(docId) : void 0;
+  const restoreRef = ctx.durability && !rebalancing ? ctx.requireManager(indexName).getRef(docId) : void 0;
   const restoreDoc = restoreRef ? structuredClone(restoreRef) : void 0;
+  let buffered = false;
   const applyRemove = async () => {
+    if (ctx.isRebalancing(indexName)) {
+      const bufferedState = ctx.bufferedDocState(indexName, docId);
+      const exists = bufferedState !== void 0 ? bufferedState === "present" : ctx.requireManager(indexName).has(docId);
+      if (!exists) {
+        throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
+      }
+    }
+    if (ctx.bufferIfRebalancing(indexName, { action: "remove", docId, indexName })) {
+      buffered = true;
+      return;
+    }
     await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
     const removeVecIndexes = ctx.requireManager(indexName).getVectorIndexes();
     removeDocumentVectors(docId, removeVecIndexes);
@@ -5282,6 +6262,9 @@ async function removeDocument(ctx, indexName, docId) {
   } else {
     await applyRemove();
   }
+  if (buffered) {
+    return;
+  }
   try {
     await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
   } catch (err) {
@@ -5343,9 +6326,6 @@ async function updateDocument(ctx, indexName, docId, document) {
   ctx.guardShutdown();
   const entry = ctx.requireIndex(indexName);
   validateDocId(docId);
-  if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
-    return;
-  }
   if (entry.config.embedding) {
     if (entry.embeddingAdapter) {
       await embedDocumentFields(
@@ -5388,7 +6368,22 @@ async function updateDocument(ctx, indexName, docId, document) {
     }
   }
   const { partitionDoc } = prepareUpdatePartitionDoc(document, updateExtractedVectors);
+  let buffered = false;
   const applyUpdate = async () => {
+    if (ctx.isRebalancing(indexName)) {
+      const bufferedState = ctx.bufferedDocState(indexName, docId);
+      const exists = bufferedState !== void 0 ? bufferedState === "present" : updateManager.has(docId);
+      if (!exists) {
+        updateManager.assertCapacity(
+          ctx.pendingRebalanceWrites(indexName),
+          ctx.rebalanceTargetPartitionCount(indexName)
+        );
+      }
+    }
+    if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
+      buffered = true;
+      return;
+    }
     await ctx.executor.execute({
       type: "update",
       indexName,
@@ -5428,6 +6423,9 @@ async function updateDocument(ctx, indexName, docId, document) {
   } else {
     await applyUpdate();
   }
+  if (buffered) {
+    return;
+  }
   try {
     await ctx.pluginRegistry.runHook("afterUpdate", {
       indexName,
@@ -5892,13 +6890,13 @@ function highlightField(text, queryTokens, language, options) {
 }
 
 // src/engine/query/highlight.ts
-function applyHighlights(hits, params, language, config) {
+function applyHighlights(hits, params, language, analysis) {
   if (!params.highlight) return;
   const queryTokenResult = tokenize(params.term ?? "", language, {
     stem: true,
     removeStopWords: true,
-    customTokenizer: config.tokenizer,
-    stopWordOverride: config.stopWords
+    customTokenizer: analysis.customTokenizer,
+    stopWordOverride: analysis.stopWords
   });
   let prefixToken;
   if (params.prefix === true && params.exact !== true && queryTokenResult.tokens.length > 0) {
@@ -5906,8 +6904,8 @@ function applyHighlights(hits, params, language, config) {
     const unstemmed = tokenize(params.term ?? "", language, {
       stem: false,
       removeStopWords: true,
-      customTokenizer: config.tokenizer,
-      stopWordOverride: config.stopWords
+      customTokenizer: analysis.customTokenizer,
+      stopWordOverride: analysis.stopWords
     });
     for (const t of unstemmed.tokens) {
       if (t.position === lastPosition) {
@@ -5930,8 +6928,64 @@ function applyHighlights(hits, params, language, config) {
         });
       }
     }
-    hit.highlights = highlights;
+    hit.highlights = highlights;
+  }
+}
+
+// src/engine/query/shared.ts
+function scoringConfigFor(params, context) {
+  const scoringMode = params.scoring ?? context.config.defaultScoring ?? "local";
+  if (scoringMode !== "broadcast") {
+    return { scoringMode };
+  }
+  return { scoringMode, globalStats: context.broadcastStats?.(context.indexName) };
+}
+function broadcastStatsForWorker(params, context, scoring) {
+  if (scoring.scoringMode !== "broadcast" || scoring.globalStats === void 0) {
+    return void 0;
+  }
+  const term = params.term;
+  if (term === void 0 || term.trim().length === 0) {
+    return void 0;
+  }
+  return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
+}
+function searchOptionsFor(manager) {
+  return {
+    bm25Params: manager.config.bm25,
+    stopWords: manager.analysis.stopWords,
+    customTokenizer: manager.analysis.customTokenizer
+  };
+}
+function collectFilterDocIds(manager, params, schema) {
+  const filterDocIds = /* @__PURE__ */ new Set();
+  if (!params.filters) return filterDocIds;
+  for (const partition of manager.getAllPartitions()) {
+    const partitionFiltered = partition.applyFilters(params.filters, schema);
+    for (const docId of partitionFiltered) {
+      filterDocIds.add(docId);
+    }
   }
+  return filterDocIds;
+}
+function vectorResultsToScored(results) {
+  return results.map((r) => ({
+    docId: r.docId,
+    score: r.score,
+    termFrequencies: {},
+    fieldLengths: {},
+    idf: {}
+  }));
+}
+function resolveVectorIndex(manager, fieldName) {
+  return manager.getVectorIndexes().get(fieldName);
+}
+function clampAlpha(alpha) {
+  if (alpha === void 0) return 0.5;
+  if (!Number.isFinite(alpha)) return 0.5;
+  if (alpha < 0) return 0;
+  if (alpha > 1) return 1;
+  return alpha;
 }
 
 // src/search/fusion.ts
@@ -6015,38 +7069,6 @@ function minMaxNormalize(docs) {
   return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
 }
 
-// src/engine/query/shared.ts
-function collectFilterDocIds(manager, params, schema) {
-  const filterDocIds = /* @__PURE__ */ new Set();
-  if (!params.filters) return filterDocIds;
-  for (const partition of manager.getAllPartitions()) {
-    const partitionFiltered = partition.applyFilters(params.filters, schema);
-    for (const docId of partitionFiltered) {
-      filterDocIds.add(docId);
-    }
-  }
-  return filterDocIds;
-}
-function vectorResultsToScored(results) {
-  return results.map((r) => ({
-    docId: r.docId,
-    score: r.score,
-    termFrequencies: {},
-    fieldLengths: {},
-    idf: {}
-  }));
-}
-function resolveVectorIndex(manager, fieldName) {
-  return manager.getVectorIndexes().get(fieldName);
-}
-function clampAlpha(alpha) {
-  if (alpha === void 0) return 0.5;
-  if (!Number.isFinite(alpha)) return 0.5;
-  if (alpha < 0) return 0;
-  if (alpha > 1) return 1;
-  return alpha;
-}
-
 // src/engine/query/vector.ts
 function executeVectorSearch(params, manager, config, limit, offset) {
   const vectorConfig = params.vector;
@@ -6075,7 +7097,8 @@ function executeVectorSearch(params, manager, config, limit, offset) {
   const scored = vectorResultsToScored(results);
   return { scored, totalMatched: scored.length };
 }
-async function executeHybridSearch(params, manager, language, config, workerSearch, indexName, limit, offset) {
+async function executeHybridSearch(params, context, limit, offset) {
+  const { manager, language, config, workerSearch, indexName } = context;
   const { vector: _vector, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
   let filterDocIds;
   if (params.filters) {
@@ -6085,22 +7108,18 @@ async function executeHybridSearch(params, manager, language, config, workerSear
     }
   }
   let textFanOutResult;
-  const textWorkerResult = workerSearch ? await workerSearch(indexName, textOnlyParams) : null;
+  const scoring = scoringConfigFor(params, context);
+  const textWorkerResult = workerSearch ? await workerSearch(indexName, textOnlyParams, broadcastStatsForWorker(textOnlyParams, context, scoring)) : null;
   if (textWorkerResult) {
     textFanOutResult = textWorkerResult;
   } else {
-    const searchOptions = {
-      bm25Params: config.bm25,
-      stopWords: config.stopWords,
-      customTokenizer: config.tokenizer
-    };
     textFanOutResult = await fanOutQuery(
       manager,
       textOnlyParams,
       language,
       config.schema,
-      { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
-      searchOptions
+      scoring,
+      searchOptionsFor(manager)
     );
   }
   const vectorConfig = params.vector;
@@ -6164,25 +7183,14 @@ async function executeQuery(params, context) {
   if (isVectorOnly && hasGlobalVectorIndex) {
     fanOutResult = executeVectorSearch(params, manager, config, limit, offset);
   } else if (isHybridMode && hasGlobalVectorIndex) {
-    fanOutResult = await executeHybridSearch(params, manager, language, config, workerSearch, indexName, limit, offset);
+    fanOutResult = await executeHybridSearch(params, context, limit, offset);
   } else {
-    const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
+    const scoring = scoringConfigFor(params, context);
+    const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring)) : null;
     if (workerResult) {
       fanOutResult = workerResult;
     } else {
-      const searchOptions = {
-        bm25Params: config.bm25,
-        stopWords: config.stopWords,
-        customTokenizer: config.tokenizer
-      };
-      fanOutResult = await fanOutQuery(
-        manager,
-        params,
-        language,
-        config.schema,
-        { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
-        searchOptions
-      );
+      fanOutResult = await fanOutQuery(manager, params, language, config.schema, scoring, searchOptionsFor(manager));
     }
   }
   const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
@@ -6233,7 +7241,7 @@ async function executeQuery(params, context) {
     }
   }
   if (params.highlight) {
-    applyHighlights(paginated, params, language, config);
+    applyHighlights(paginated, params, language, manager.analysis);
   }
   const elapsed = now() - startTime;
   return {
@@ -6261,34 +7269,21 @@ async function executePreflight(params, context) {
     const result = executeVectorSearch(params, manager, config, preflightLimit, preflightOffset);
     totalMatched = result.totalMatched;
   } else if (isHybridMode && hasGlobalVectorIndex) {
-    const result = await executeHybridSearch(
-      params,
-      manager,
-      language,
-      config,
-      workerSearch,
-      indexName,
-      preflightLimit,
-      preflightOffset
-    );
+    const result = await executeHybridSearch(params, context, preflightLimit, preflightOffset);
     totalMatched = result.totalMatched;
   } else {
-    const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
+    const scoring = scoringConfigFor(params, context);
+    const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring)) : null;
     if (workerResult) {
       totalMatched = workerResult.totalMatched;
     } else {
-      const searchOptions = {
-        bm25Params: config.bm25,
-        stopWords: config.stopWords,
-        customTokenizer: config.tokenizer
-      };
       const fanOutResult = await fanOutQuery(
         manager,
         params,
         language,
         config.schema,
-        { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
-        searchOptions
+        scoring,
+        searchOptionsFor(manager)
       );
       totalMatched = fanOutResult.totalMatched;
     }
@@ -6300,6 +7295,7 @@ async function executePreflight(params, context) {
 // src/partitioning/write-ahead-queue.ts
 function createWriteAheadQueue(maxSize = 1e4) {
   const entries = [];
+  const docStates = /* @__PURE__ */ new Map();
   let nextSequence = 1;
   return {
     push(entry) {
@@ -6312,6 +7308,7 @@ function createWriteAheadQueue(maxSize = 1e4) {
       }
       const seq = nextSequence++;
       entries.push({ ...entry, sequenceNumber: seq });
+      docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
       return seq;
     },
     drain() {
@@ -6321,6 +7318,10 @@ function createWriteAheadQueue(maxSize = 1e4) {
     },
     clear() {
       entries.length = 0;
+      docStates.clear();
+    },
+    bufferedDocState(docId) {
+      return docStates.get(docId);
     },
     get size() {
       return entries.length;
@@ -6332,6 +7333,135 @@ function createWriteAheadQueue(maxSize = 1e4) {
 }
 
 // src/engine/rebalance-executor.ts
+function warnHookError(hook, err) {
+  console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
+}
+function applyReplayUpdateVectors(entry, vectorFieldPaths) {
+  const updateVectors = /* @__PURE__ */ new Map();
+  for (const fieldPath of vectorFieldPaths) {
+    updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
+  }
+  let updateDoc = entry.document;
+  if (updateVectors.size > 0) {
+    updateDoc = structuredClone(entry.document);
+    for (const fieldPath of updateVectors.keys()) {
+      deleteNestedValue(updateDoc, fieldPath);
+    }
+  }
+  return { updateDoc, updateVectors };
+}
+async function replayEntry(manager, indexName, entry, ctx) {
+  const vecIndexes = manager.getVectorIndexes();
+  const vectorFieldPaths = vecIndexes.size > 0 ? ctx.requireIndex(indexName).vectorFieldPaths : /* @__PURE__ */ new Set();
+  if (entry.action === "insert" && entry.document) {
+    const { partitionDoc, extractedVectors } = prepareDocumentVectors(
+      entry.document,
+      vectorFieldPaths,
+      vecIndexes
+    );
+    const overwrote = manager.has(entry.docId);
+    if (overwrote) {
+      removeDocumentVectors(entry.docId, vecIndexes);
+      manager.remove(entry.docId);
+    }
+    manager.insert(entry.docId, partitionDoc);
+    if (extractedVectors.size > 0) {
+      insertDocumentVectors(entry.docId, extractedVectors, vecIndexes);
+      for (const fieldPath of extractedVectors.keys()) {
+        vecIndexes.get(fieldPath)?.scheduleBuild();
+      }
+    }
+    try {
+      await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
+    } catch (err) {
+      warnHookError("afterInsert", err);
+    }
+    await ctx.orchestrator.replicateToWorkers({
+      type: overwrote ? "update" : "insert",
+      indexName,
+      docId: entry.docId,
+      document: entry.document,
+      requestId: `replicate-insert-${entry.docId}`
+    });
+    return;
+  }
+  if (entry.action === "remove") {
+    manager.remove(entry.docId);
+    removeDocumentVectors(entry.docId, vecIndexes);
+    try {
+      await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
+    } catch (err) {
+      warnHookError("afterRemove", err);
+    }
+    await ctx.orchestrator.replicateToWorkers({
+      type: "remove",
+      indexName,
+      docId: entry.docId,
+      requestId: `replicate-remove-${entry.docId}`
+    });
+    return;
+  }
+  if (entry.action === "update" && entry.document) {
+    const oldDocument = manager.get(entry.docId);
+    const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
+    if (manager.has(entry.docId)) {
+      manager.update(entry.docId, updateDoc);
+    } else {
+      manager.insert(entry.docId, updateDoc);
+    }
+    for (const [fieldPath, newVec] of updateVectors) {
+      const vecIndex = vecIndexes.get(fieldPath);
+      if (!vecIndex) continue;
+      if (newVec === null) {
+        if (vecIndex.has(entry.docId)) {
+          vecIndex.remove(entry.docId);
+        }
+      } else {
+        const oldVec = vecIndex.getVector(entry.docId);
+        if (!vectorsEqual(oldVec, newVec)) {
+          if (vecIndex.has(entry.docId)) {
+            vecIndex.remove(entry.docId);
+          }
+          vecIndex.insert(entry.docId, newVec);
+          vecIndex.scheduleBuild();
+        }
+      }
+    }
+    try {
+      await ctx.pluginRegistry.runHook("afterUpdate", {
+        indexName,
+        docId: entry.docId,
+        oldDocument: oldDocument ?? {},
+        newDocument: entry.document
+      });
+    } catch (err) {
+      warnHookError("afterUpdate", err);
+    }
+    await ctx.orchestrator.replicateToWorkers({
+      type: "update",
+      indexName,
+      docId: entry.docId,
+      document: entry.document,
+      requestId: `replicate-update-${entry.docId}`
+    });
+  }
+}
+async function replayQueued(manager, indexName, waq, ctx) {
+  while (true) {
+    const entries = waq.drain();
+    if (entries.length === 0) break;
+    for (const entry of entries) {
+      try {
+        await replayEntry(manager, indexName, entry, ctx);
+      } catch (replayErr) {
+        const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
+        if (!isMissing) {
+          console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
+        }
+      }
+    }
+  }
+}
 async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
   if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
     throw new NarsilError(
@@ -6340,6 +7470,15 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
       { targetPartitionCount }
     );
   }
+  const entry = ctx.requireIndex(indexName);
+  const maxPartitions = entry.config.partitions?.maxPartitions;
+  if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
+    throw new NarsilError(
+      ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+      `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
+      { targetPartitionCount, maxPartitions }
+    );
+  }
   if (ctx.rebalancingIndexes.has(indexName)) {
     throw new NarsilError(
       ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
@@ -6352,6 +7491,8 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
   const waq = createWriteAheadQueue();
   ctx.waqMap.set(indexName, waq);
   ctx.rebalancingIndexes.add(indexName);
+  ctx.rebalanceTargets.set(indexName, targetPartitionCount);
+  const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
   try {
     const oldCount = manager.partitionCount;
     await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
@@ -6364,71 +7505,36 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
         }
       }
     });
-    const entries = waq.drain();
-    const appliedSeqs = ctx.lastAppliedSeqMap.get(indexName) ?? /* @__PURE__ */ new Map();
-    const rebalanceEntry = ctx.requireIndex(indexName);
-    const rebalanceVecIndexes = manager.getVectorIndexes();
-    const rebalanceVectorFieldPaths = rebalanceVecIndexes.size > 0 ? rebalanceEntry.vectorFieldPaths : /* @__PURE__ */ new Set();
-    for (const entry of entries) {
-      const partitionLastSeq = appliedSeqs.get(0) ?? 0;
-      if (entry.sequenceNumber <= partitionLastSeq) continue;
-      try {
-        if (entry.action === "insert" && entry.document) {
-          const { partitionDoc, extractedVectors } = prepareDocumentVectors(
-            entry.document,
-            rebalanceVectorFieldPaths,
-            rebalanceVecIndexes
-          );
-          manager.insert(entry.docId, partitionDoc);
-          if (extractedVectors.size > 0) {
-            insertDocumentVectors(entry.docId, extractedVectors, rebalanceVecIndexes);
-          }
-        } else if (entry.action === "remove") {
-          manager.remove(entry.docId);
-          removeDocumentVectors(entry.docId, rebalanceVecIndexes);
-        } else if (entry.action === "update" && entry.document) {
-          const replayUpdateVectors = /* @__PURE__ */ new Map();
-          for (const fieldPath of rebalanceVectorFieldPaths) {
-            const newVec = extractVectorFromDoc(entry.document, fieldPath);
-            replayUpdateVectors.set(fieldPath, newVec);
-          }
-          const replayUpdateDoc = replayUpdateVectors.size > 0 ? structuredClone(entry.document) : entry.document;
-          if (replayUpdateVectors.size > 0) {
-            for (const fieldPath of replayUpdateVectors.keys()) {
-              deleteNestedValue(replayUpdateDoc, fieldPath);
-            }
-          }
-          manager.update(entry.docId, replayUpdateDoc);
-          for (const [fieldPath, newVec] of replayUpdateVectors) {
-            const vecIndex = rebalanceVecIndexes.get(fieldPath);
-            if (!vecIndex) continue;
-            if (newVec === null) {
-              if (vecIndex.has(entry.docId)) {
-                vecIndex.remove(entry.docId);
-              }
-            } else {
-              const oldVec = vecIndex.getVector(entry.docId);
-              if (!vectorsEqual(oldVec, newVec)) {
-                vecIndex.remove(entry.docId);
-                vecIndex.insert(entry.docId, newVec);
-              }
-            }
-          }
-        }
-      } catch (replayErr) {
-        const isDuplicate = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_ALREADY_EXISTS;
-        const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
-        if (!isDuplicate && !isMissing) {
-          console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
-        }
-      }
-      appliedSeqs.set(0, entry.sequenceNumber);
+    await replayQueued(manager, indexName, waq, ctx);
+    if (ctx.durabilityManager) {
+      await ctx.durabilityManager.persistMetadata(indexName);
+      await ctx.durabilityManager.checkpoint(indexName);
+      await replayQueued(manager, indexName, waq, ctx);
+    }
+    await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
+    do {
+      await replayQueued(manager, indexName, waq, ctx);
+    } while (waq.size > 0);
+  } catch (err) {
+    try {
+      await replayQueued(manager, indexName, waq, ctx);
+      await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
+      do {
+        await replayQueued(manager, indexName, waq, ctx);
+      } while (waq.size > 0);
+    } catch (recoveryErr) {
+      console.warn(
+        `Rebalance failure recovery for index "${indexName}" did not complete:`,
+        recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
+      );
     }
-    ctx.lastAppliedSeqMap.set(indexName, appliedSeqs);
+    throw err;
   } finally {
     ctx.rebalancingIndexes.delete(indexName);
+    ctx.rebalanceTargets.delete(indexName);
     ctx.waqMap.delete(indexName);
   }
+  ctx.checkWatermark(indexName);
 }
 
 // src/engine/resolve-vector-text.ts
@@ -6474,8 +7580,176 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
   return { ...params, vector: resolved };
 }
 
-// src/engine/snapshot.ts
+// src/engine/snapshot/restore-config.ts
+function invalid(message) {
+  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
+}
+function isPlainObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function restoredBm25(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw)) {
+    invalid("bm25 must be an object");
+  }
+  const { k1, b } = raw;
+  if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
+    invalid("bm25 k1 must be a finite number");
+  }
+  if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
+    invalid("bm25 b must be a finite number");
+  }
+  return {
+    ...k1 !== void 0 ? { k1 } : {},
+    ...b !== void 0 ? { b } : {}
+  };
+}
+function restoredStopWords(name, list) {
+  if (name !== void 0 && typeof name !== "string") {
+    invalid("stopWords must be a name");
+  }
+  if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
+    invalid("stopWordList must be a list of words");
+  }
+  if (typeof name === "string" && Array.isArray(list)) {
+    invalid("stopWords and stopWordList cannot both be present");
+  }
+  if (typeof name === "string") {
+    return name;
+  }
+  if (Array.isArray(list)) {
+    return new Set(list);
+  }
+  return void 0;
+}
+function restoredPartitionConfig(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw)) {
+    invalid("partitionConfig must be an object");
+  }
+  const { maxDocsPerPartition, maxPartitions, watermark } = raw;
+  if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
+    invalid("partitionConfig maxDocsPerPartition must be a positive integer");
+  }
+  if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
+    invalid("partitionConfig maxPartitions must be a positive integer");
+  }
+  if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
+    invalid("partitionConfig watermark must be above 0 and at most 1");
+  }
+  return {
+    ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
+    ...typeof maxPartitions === "number" ? { maxPartitions } : {},
+    ...typeof watermark === "number" ? { watermark } : {}
+  };
+}
+function restoredScoring(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
+    invalid("defaultScoring must be 'local', 'dfs', or 'broadcast'");
+  }
+  return raw;
+}
+function restoredBoolean(raw, field) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (typeof raw !== "boolean") {
+    invalid(`${field} must be a boolean`);
+  }
+  return raw;
+}
+function restoredRequired(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
+    invalid("required must be a list of field paths");
+  }
+  return raw;
+}
+function restoredVectorPromotion(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw) || raw.hnswConfig !== void 0 && !isPlainObject(raw.hnswConfig)) {
+    invalid("vectorPromotion must be an object");
+  }
+  return raw;
+}
+function restoredEmbedding(raw) {
+  if (raw === void 0) {
+    return void 0;
+  }
+  if (!isPlainObject(raw)) {
+    invalid("embedding must be an object");
+  }
+  const { fields, adapter } = raw;
+  if (adapter !== void 0 && typeof adapter !== "string") {
+    invalid("embedding adapter must be a name");
+  }
+  if (!isPlainObject(fields)) {
+    invalid("embedding fields must be an object");
+  }
+  for (const value of Object.values(fields)) {
+    const isPath = typeof value === "string";
+    const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
+    if (!isPath && !isPathList) {
+      invalid("embedding fields must map vector fields to source paths");
+    }
+  }
+  return {
+    fields,
+    ...adapter !== void 0 ? { adapter } : {}
+  };
+}
+function restoredConfigFields(envelope) {
+  if (envelope.tokenizer !== void 0 && typeof envelope.tokenizer !== "string") {
+    invalid("tokenizer must be a name");
+  }
+  const stopWords = restoredStopWords(envelope.stopWords, envelope.stopWordList);
+  const bm25 = restoredBm25(envelope.bm25);
+  const surfaceForms = restoredBoolean(envelope.surfaceForms, "surfaceForms");
+  const partitions = restoredPartitionConfig(envelope.partitionConfig);
+  const defaultScoring = restoredScoring(envelope.defaultScoring);
+  const trackPositions = restoredBoolean(envelope.trackPositions, "trackPositions");
+  const strict = restoredBoolean(envelope.strict, "strict");
+  const required = restoredRequired(envelope.required);
+  const vectorPromotion = restoredVectorPromotion(envelope.vectorPromotion);
+  return {
+    ...typeof envelope.tokenizer === "string" ? { tokenizer: envelope.tokenizer } : {},
+    ...stopWords !== void 0 ? { stopWords } : {},
+    ...bm25 !== void 0 ? { bm25 } : {},
+    ...surfaceForms !== void 0 ? { surfaceForms } : {},
+    ...partitions !== void 0 ? { partitions } : {},
+    ...defaultScoring !== void 0 ? { defaultScoring } : {},
+    ...trackPositions !== void 0 ? { trackPositions } : {},
+    ...strict !== void 0 ? { strict } : {},
+    ...required !== void 0 ? { required } : {},
+    ...vectorPromotion !== void 0 ? { vectorPromotion } : {}
+  };
+}
+
+// src/engine/snapshot/index.ts
 async function createSnapshot(manager, entry) {
+  if (entry.config.tokenizer !== void 0 && typeof entry.config.tokenizer !== "string") {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      "A snapshot persists no code, so the index tokenizer must be a name registered with registerTokenizer"
+    );
+  }
+  if (typeof entry.config.stopWords === "function") {
+    throw new NarsilError(
+      ErrorCodes.CONFIG_INVALID,
+      "A snapshot persists no code, so the index stop words must be a name registered with registerStopWords"
+    );
+  }
   const partitionBuffers = [];
   for (let i = 0; i < manager.partitionCount; i++) {
     partitionBuffers.push(manager.serializePartitionToBytes(i));
@@ -6485,16 +7759,41 @@ async function createSnapshot(manager, entry) {
   for (const [fieldPath, vecIndex] of snapshotVecIndexes) {
     vectorPayloads[fieldPath] = vecIndex.serialize();
   }
+  const config = entry.config;
+  const bm25 = config.bm25;
   const { encode: encode10 } = await import('@msgpack/msgpack');
   return encode10({
     version: 2,
-    schema: entry.config.schema,
+    schema: config.schema,
     language: entry.language.name,
+    analysisRevision: entry.language.revision,
+    ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
+    ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
+    ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort() } : {},
+    ...bm25 !== void 0 ? {
+      bm25: {
+        ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
+        ...bm25.b !== void 0 ? { b: bm25.b } : {}
+      }
+    } : {},
+    ...config.surfaceForms === true ? { surfaceForms: true } : {},
+    ...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
+    ...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
+    ...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
+    ...config.strict !== void 0 ? { strict: config.strict } : {},
+    ...config.required !== void 0 ? { required: config.required } : {},
+    ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
+    ...config.embedding !== void 0 ? {
+      embedding: {
+        fields: config.embedding.fields,
+        ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
+      }
+    } : {},
     partitions: partitionBuffers,
     vectorIndexes: vectorPayloads
   });
 }
-async function restoreFromSnapshot(indexName, data, executor, indexRegistry, getVectorFieldPaths2, dropIndex, requireManager) {
+async function restoreFromSnapshot(indexName, data, deps) {
   if (!(data instanceof Uint8Array)) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
   }
@@ -6516,24 +7815,60 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
   if (!Array.isArray(envelope.partitions)) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing partitions");
   }
+  const configFields = restoredConfigFields(envelope);
+  const embedding = restoredEmbedding(envelope.embedding);
   const language = getLanguage(envelope.language);
   const schema = envelope.schema;
   validateSchema(schema);
-  if (indexRegistry.has(indexName)) {
-    await dropIndex(indexName);
+  validateVectorPromotion(configFields.vectorPromotion);
+  if (configFields.required !== void 0 && configFields.required.length > 0) {
+    validateRequiredFieldsInSchema(configFields.required, schema);
+  }
+  const indexConfig = {
+    schema,
+    language: envelope.language,
+    ...configFields,
+    ...embedding !== void 0 ? {
+      embedding: {
+        fields: embedding.fields,
+        ...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
+      }
+    } : {}
+  };
+  resolveIndexAnalysis(indexConfig);
+  const adapterName = embedding?.adapter ?? null;
+  let embeddingAdapter = null;
+  if (embedding !== void 0) {
+    const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
+    if (candidate) {
+      validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
+      embeddingAdapter = candidate;
+    }
+  }
+  if (deps.indexRegistry.has(indexName)) {
+    await deps.dropIndex(indexName);
+  }
+  const storedRevision = typeof envelope.analysisRevision === "string" ? envelope.analysisRevision : null;
+  deps.clearAnalysisStale(indexName);
+  if (storedRevision !== language.revision) {
+    deps.markAnalysisStale({
+      indexName,
+      language: language.name,
+      storedRevision,
+      currentRevision: language.revision
+    });
   }
-  const indexConfig = { schema, language: envelope.language };
-  executor.createIndex(indexName, indexConfig, language);
-  const vectorFieldPaths = getVectorFieldPaths2(schema);
-  indexRegistry.set(indexName, {
+  deps.executor.createIndex(indexName, indexConfig, language);
+  const vectorFieldPaths = deps.getVectorFieldPaths(schema);
+  deps.indexRegistry.set(indexName, {
     config: indexConfig,
     language,
-    embeddingAdapter: null,
-    embeddingAdapterName: null,
+    embeddingAdapter,
+    embeddingAdapterName: adapterName,
     vectorFieldPaths
   });
   try {
-    const manager = requireManager(indexName);
+    const manager = deps.requireManager(indexName);
     while (manager.partitionCount < envelope.partitions.length) {
       manager.addPartition();
     }
@@ -6553,12 +7888,27 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
     }
   } catch (err) {
     try {
-      executor.dropIndex(indexName);
-      indexRegistry.delete(indexName);
+      deps.executor.dropIndex(indexName);
+      deps.indexRegistry.delete(indexName);
     } catch (_) {
     }
     throw err;
   }
+  if (deps.durability) {
+    const manager = deps.requireManager(indexName);
+    for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
+      for (const docId of manager.getPartition(partitionId).docIds()) {
+        const document = manager.get(docId);
+        if (document !== void 0) {
+          await deps.durability.recordInsertOrUpdate(indexName, docId, document, noopApply);
+        }
+      }
+    }
+    await deps.durability.manager.persistMetadata(indexName);
+    await deps.durability.manager.checkpoint(indexName);
+  }
+}
+function noopApply() {
 }
 
 // src/engine/suggest.ts
@@ -6679,6 +8029,10 @@ async function createNarsil(config) {
   if (core.durability) {
     await core.durability.manager.recover();
   }
+  if (core.invalidation) {
+    await core.invalidation.start();
+  }
+  await core.analysisRebuild.reviewStaleIndexes();
   return createNarsilFromCore(core, config);
 }
 function createNarsilFromCore(core, config) {
@@ -6698,6 +8052,8 @@ function createNarsilFromCore(core, config) {
     mutationCtx,
     rebalanceCtx
   } = core;
+  const invalidation = core.invalidation;
+  const broadcastStats = invalidation === null ? void 0 : (name) => invalidation.broadcastStats(name);
   const narsil = {
     createIndex(name, indexConfig) {
       return createEngineIndex(core, config, name, indexConfig);
@@ -6716,7 +8072,8 @@ function createNarsilFromCore(core, config) {
           name,
           documentCount: manager?.countDocuments() ?? 0,
           partitionCount: manager?.partitionCount ?? 0,
-          language: entry.language.name
+          language: entry.language.name,
+          ...core.analysisRebuild.isStale(name) ? { analysisStale: true } : {}
         });
       }
       return infos;
@@ -6798,7 +8155,8 @@ function createNarsilFromCore(core, config) {
         language: entry.language,
         config: entry.config,
         workerSearch,
-        indexName
+        indexName,
+        broadcastStats
       });
       try {
         await pluginRegistry.runHook("afterSearch", {
@@ -6809,6 +8167,9 @@ function createNarsilFromCore(core, config) {
       } catch (err) {
         console.warn("afterSearch plugin hook error:", err);
       }
+      if (core.analysisRebuild.isStale(indexName)) {
+        result.analysisStale = true;
+      }
       return result;
     },
     async preflight(indexName, params) {
@@ -6822,17 +8183,31 @@ function createNarsilFromCore(core, config) {
         entry.embeddingAdapterName
       );
       const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
-      return executePreflight(resolvedParams, {
+      const result = await executePreflight(resolvedParams, {
         manager,
         language: entry.language,
         config: entry.config,
         workerSearch,
-        indexName
+        indexName,
+        broadcastStats
       });
+      if (core.analysisRebuild.isStale(indexName)) {
+        result.analysisStale = true;
+      }
+      return result;
     },
     async suggest(indexName, params) {
       guardShutdown();
-      return executeSuggest(requireManager(indexName), requireIndex(indexName).language, params);
+      const result = await executeSuggest(requireManager(indexName), requireIndex(indexName).language, params);
+      if (core.analysisRebuild.isStale(indexName)) {
+        result.analysisStale = true;
+      }
+      return result;
+    },
+    async rebuildAnalysis(indexName) {
+      guardShutdown();
+      requireIndex(indexName);
+      await core.analysisRebuild.rebuild(indexName);
     },
     async snapshot(indexName) {
       guardShutdown();
@@ -6840,15 +8215,18 @@ function createNarsilFromCore(core, config) {
     },
     async restore(indexName, data) {
       guardShutdown();
-      return restoreFromSnapshot(
-        indexName,
-        data,
+      return restoreFromSnapshot(indexName, data, {
         executor,
         indexRegistry,
         getVectorFieldPaths,
-        narsil.dropIndex.bind(narsil),
-        requireManager
-      );
+        dropIndex: narsil.dropIndex.bind(narsil),
+        requireManager,
+        durability,
+        embeddingAdapters: core.embeddingAdapters,
+        defaultEmbeddingAdapter: config?.embedding ?? null,
+        markAnalysisStale: core.analysisRebuild.markStale,
+        clearAnalysisStale: core.analysisRebuild.clearStale
+      });
     },
     async checkpoint(indexName) {
       guardShutdown();
@@ -6862,6 +8240,7 @@ function createNarsilFromCore(core, config) {
       requireIndex(indexName);
       await executor.execute({ type: "clear", indexName, requestId: indexName });
       await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
+      core.watermarkNotifier.forget(indexName);
     },
     async rebalance(indexName, targetPartitionCount) {
       guardShutdown();
@@ -6878,11 +8257,18 @@ function createNarsilFromCore(core, config) {
           `Index "${indexName}" is currently being rebalanced`
         );
       }
+      validatePartitionConfig(partitionConfig);
+      if (partitionConfig.maxPartitions !== void 0 && partitionConfig.maxPartitions < manager.partitionCount) {
+        throw new NarsilError(
+          ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
+          `maxPartitions (${partitionConfig.maxPartitions}) is less than the current partition count (${manager.partitionCount})`,
+          { maxPartitions: partitionConfig.maxPartitions, partitionCount: manager.partitionCount }
+        );
+      }
       const currentDocCount = manager.countDocuments();
       const newMaxDocs = partitionConfig.maxDocsPerPartition ?? entry.config.partitions?.maxDocsPerPartition;
-      const newMaxPartitions = partitionConfig.maxPartitions ?? entry.config.partitions?.maxPartitions ?? manager.partitionCount;
       if (newMaxDocs !== void 0) {
-        const newTotalCapacity = newMaxDocs * newMaxPartitions;
+        const newTotalCapacity = newMaxDocs * manager.partitionCount;
         if (newTotalCapacity < currentDocCount) {
           throw new NarsilError(
             ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
@@ -6896,6 +8282,9 @@ function createNarsilFromCore(core, config) {
         entry.config.partitions.maxDocsPerPartition = partitionConfig.maxDocsPerPartition;
       if (partitionConfig.maxPartitions !== void 0)
         entry.config.partitions.maxPartitions = partitionConfig.maxPartitions;
+      if (partitionConfig.watermark !== void 0) entry.config.partitions.watermark = partitionConfig.watermark;
+      if (durability) await durability.manager.persistMetadata(indexName);
+      core.watermarkNotifier.check(indexName);
     },
     async getMemoryStats() {
       let estimatedIndexBytes = 0;