@delali/narsil 0.3.1-next.72 → 0.3.1-next.74

package/dist/distribution.mjs

@@ -1,27 +1,33 @@
 import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
-import { validateIndexName, indexConfigKey, indexSettingsOf, putIndexMetadata, withPartitionCount, getIndexMetadata, getClusterIndexConfig } from './chunk-Y52WZWAU.mjs';
-import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-7QMTA7AA.mjs';
-import { now, listBindingOf, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-2X2ZQOAB.mjs';
+import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-TVFLHM7M.mjs';
+import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-CUZOX6CN.mjs';
+import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-CTOSGRM7.mjs';
 import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
-import { providedDocId, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-CVPA5Q7R.mjs';
-import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-XZ4PAREY.mjs';
-import { generateId, deserializePayloadV2, createPartitionIndex, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-HVJYTKK7.mjs';
-import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, MAX_FILTER_DEPTH, filterTestCount, MAX_FILTER_TESTS, fnv1aUint32s, mergePartitionStats, MAX_FILTER_ARRAY_SIZE } from './chunk-QR4OKRVY.mjs';
-import { getLanguage } from './chunk-KZTI25D3.mjs';
-import { validateSchema, DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH } from './chunk-M6XF5HDE.mjs';
+import { providedDocId, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-LX4JERXI.mjs';
+import { generateId, decodeVectorFilePayload, bytesToVectors, isDead, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-BVBIQQKG.mjs';
+import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, MAX_FILTER_DEPTH, filterTestCount, MAX_FILTER_TESTS, fnv1aUint32s, mergePartitionStats, MAX_FILTER_ARRAY_SIZE } from './chunk-VKX2BPXD.mjs';
+import { getLanguage } from './chunk-4MQ5F4IL.mjs';
+import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, DEFAULT_PAGE_SIZE, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_SORT_FIELDS, MAX_DOC_ID_LENGTH, MAX_CURSOR_LENGTH, MAX_TOLERANCE, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN, MAX_PREFIX_LENGTH, RESULT_WINDOW } from './chunk-3DDUNI6N.mjs';
 import './chunk-TGP5M5R7.mjs';
-import './chunk-LCCL6EEH.mjs';
+import './chunk-PO53FV4Q.mjs';
 import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
-import './chunk-P2ZZN32C.mjs';
-import { crc32 } from './chunk-3L4U6EJQ.mjs';
-import './chunk-IFBKLBGB.mjs';
-import './chunk-3HQPUCCT.mjs';
+import './chunk-2E35UM3G.mjs';
+import './chunk-XW25X6AU.mjs';
 import './chunk-ZKRV5M2P.mjs';
+import './chunk-T5ZWEWVD.mjs';
+import './chunk-I23G3IZC.mjs';
+import { validateSchema, requireCreatableSchema, encodeDocument, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH } from './chunk-Z3DQPOJQ.mjs';
+import './chunk-KEHK2O6P.mjs';
 import './chunk-U2BKYEC7.mjs';
 import { compareCodePoints, toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, exceedsCodePoints, compareSortValues } from './chunk-LJYTW7GT.mjs';
 import './chunk-POWOTUKJ.mjs';
-import { ErrorCodes, NarsilError, describeError } from './chunk-RV77FR2F.mjs';
+import { ErrorCodes, NarsilError, describeError } from './chunk-I64LXKO3.mjs';
 import { encode, decode } from '@msgpack/msgpack';
+import { randomUUID } from 'crypto';
+import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
+import { mkdtemp, rm, mkdir } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
 
 // src/distribution/cluster/constants.ts
 var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
@@ -1251,7 +1257,6 @@ var REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES = 40;
 var DEFAULT_LOG_RETENTION_BYTES = 268435456;
 var INSYNC_CAS_ATTEMPTS = 5;
 var SNAPSHOT_CHUNK_SIZE = 65536;
-var MAX_SNAPSHOT_SIZE_BYTES = 2 * 1024 * 1024 * 1024;
 
 // src/distribution/replication/insync.ts
 var REFUSE = { outcome: "refuse" };
@@ -2516,6 +2521,119 @@ function createDataNodeLifecycle(config) {
   };
   return handle;
 }
+var SPOOL_FILE_PREFIX = "narsil-snapshot-";
+var DURABLE_SPOOL_DIRECTORY = ".snapshot-transfer";
+var TEMPORARY_SPOOL_DIRECTORY_PREFIX = "narsil-snapshot-transfer-";
+var OWNER_ONLY_FILE_MODE = 384;
+var OWNER_ONLY_DIRECTORY_MODE = 448;
+function defaultSpoolDirectory() {
+  return tmpdir();
+}
+function spoolFilePath(directory) {
+  return join(directory, `${SPOOL_FILE_PREFIX}${randomUUID()}`);
+}
+async function* readSpoolFile(path) {
+  const descriptor = openSync(path, "r");
+  try {
+    for (; ; ) {
+      const buffer = new Uint8Array(SNAPSHOT_CHUNK_SIZE);
+      const bytesRead = readSync(descriptor, buffer, 0, buffer.byteLength, null);
+      if (bytesRead === 0) return;
+      yield bytesRead === buffer.byteLength ? buffer : buffer.subarray(0, bytesRead);
+    }
+  } finally {
+    closeSync(descriptor);
+  }
+}
+function spooledFile(path, totalBytes, checksum) {
+  let discarded = false;
+  return {
+    totalBytes,
+    checksum,
+    read: () => readSpoolFile(path),
+    async discard() {
+      if (discarded) return;
+      discarded = true;
+      removeQuietly(path);
+    }
+  };
+}
+function openSnapshotSpoolWriter(directory) {
+  const path = spoolFilePath(directory);
+  let descriptor = openSync(path, "wx", OWNER_ONLY_FILE_MODE);
+  let checksumState = crc32Init();
+  let totalBytes = 0;
+  const close = () => {
+    if (descriptor === null) return;
+    const closing = descriptor;
+    descriptor = null;
+    closeSync(closing);
+  };
+  return {
+    write(bytes) {
+      if (descriptor === null) {
+        throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "The snapshot spool file is already closed", {
+          path
+        });
+      }
+      let offset = 0;
+      while (offset < bytes.byteLength) {
+        offset += writeSync(descriptor, bytes, offset, bytes.byteLength - offset);
+      }
+      checksumState = crc32Update(checksumState, bytes);
+      totalBytes += bytes.byteLength;
+    },
+    checksum() {
+      return crc32Final(checksumState);
+    },
+    finish() {
+      close();
+      return spooledFile(path, totalBytes, crc32Final(checksumState));
+    },
+    discard() {
+      try {
+        close();
+      } finally {
+        removeQuietly(path);
+      }
+    }
+  };
+}
+function removeQuietly(path) {
+  try {
+    rmSync(path, { force: true });
+  } catch (_) {
+  }
+}
+async function spoolSnapshotStream(directory, stream) {
+  const reader = stream.getReader();
+  let writer = null;
+  try {
+    writer = openSnapshotSpoolWriter(directory);
+    for (; ; ) {
+      const step = await reader.read();
+      if (step.done) break;
+      if (step.value.byteLength > 0) writer.write(step.value);
+    }
+    const spooled = writer.finish();
+    writer = null;
+    return spooled;
+  } catch (err) {
+    writer?.discard();
+    await reader.cancel().catch(() => void 0);
+    throw err;
+  } finally {
+    reader.releaseLock();
+  }
+}
+async function openSnapshotSpoolDirectory(durabilityDirectory) {
+  const path = durabilityDirectory === void 0 ? await mkdtemp(join(tmpdir(), TEMPORARY_SPOOL_DIRECTORY_PREFIX)) : join(durabilityDirectory, DURABLE_SPOOL_DIRECTORY);
+  if (durabilityDirectory !== void 0) {
+    await rm(path, { recursive: true, force: true });
+    await mkdir(path, { recursive: true, mode: OWNER_ONLY_DIRECTORY_MODE });
+  }
+  return { path, dispose: () => rm(path, { recursive: true, force: true }) };
+}
 
 // src/distribution/cluster-node/local-index-gone.ts
 function localIndexIsGone(err) {
@@ -2583,6 +2701,19 @@ function fail(state, code, message, details = {}) {
     return;
   }
   state.failure = { ok: false, code, message, details };
+  releaseSpool(state);
+}
+function releaseSpool(state) {
+  const spool = state.spool;
+  if (spool === null) return;
+  state.spool = null;
+  spool.discard();
+}
+function spoolFailure(state, err) {
+  const cause = err instanceof Error ? err.message : String(err);
+  fail(state, ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `the snapshot could not be written to local storage: ${cause}`, {
+    cause
+  });
 }
 function applyErrorEnvelope(state, record) {
   const code = typeof record.code === "string" && record.code.length > 0 ? record.code : "UNKNOWN_ERROR";
@@ -2633,19 +2764,17 @@ function applyStartPayload(state, record) {
     fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "totalBytes must be a non-negative integer");
     return;
   }
-  if (start.totalBytes > MAX_SNAPSHOT_SIZE_BYTES) {
-    fail(state, ErrorCodes.SNAPSHOT_SYNC_TOO_LARGE, `totalBytes ${start.totalBytes} exceeds the limit`, {
-      totalBytes: start.totalBytes,
-      limit: MAX_SNAPSHOT_SIZE_BYTES
-    });
+  try {
+    state.spool = openSnapshotSpoolWriter(state.expectation.spoolDirectory);
+  } catch (err) {
+    spoolFailure(state, err);
     return;
   }
   state.header = header;
   state.totalBytes = start.totalBytes;
-  state.assembled = new Uint8Array(start.totalBytes);
 }
 function applyChunkPayload(state, payload) {
-  if (state.totalBytes === null || state.assembled === null || state.header === null) {
+  if (state.totalBytes === null || state.header === null) {
     fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received chunk before SNAPSHOT_START");
     return;
   }
@@ -2699,7 +2828,16 @@ function applyChunkPayload(state, payload) {
     });
     return;
   }
-  state.assembled.set(payload.data, payload.offset);
+  if (state.spool === null) {
+    fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received chunk after the snapshot stream closed");
+    return;
+  }
+  try {
+    state.spool.write(payload.data);
+  } catch (err) {
+    spoolFailure(state, err);
+    return;
+  }
   state.accumulatedBytes = nextBytes;
   state.expectedNextOffset = payload.offset + payload.data.byteLength;
 }
@@ -2777,7 +2915,7 @@ function createSnapshotStreamState(expectation) {
     expectation,
     header: null,
     totalBytes: null,
-    assembled: null,
+    spool: null,
     accumulatedBytes: 0,
     expectedNextOffset: 0,
     endPayload: null,
@@ -2832,11 +2970,20 @@ function handleIncomingSnapshotRecord(state, record) {
   }
   fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "unrecognised snapshot frame shape");
 }
+function abandonSnapshotStream(state) {
+  fail(state, ErrorCodes.SNAPSHOT_SYNC_ABORTED, "the snapshot stream was abandoned before it finished");
+  releaseSpool(state);
+}
 function finalizeSnapshotStream(state) {
+  const result = settleSnapshotStream(state);
+  if (!result.ok) releaseSpool(state);
+  return result;
+}
+function settleSnapshotStream(state) {
   if (state.failure !== null) {
     return state.failure;
   }
-  if (state.header === null || state.totalBytes === null || state.assembled === null) {
+  if (state.header === null || state.totalBytes === null) {
     return {
       ok: false,
       code: ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING,
@@ -2868,9 +3015,28 @@ function finalizeSnapshotStream(state) {
       }
     };
   }
+  const spool = state.spool;
+  if (spool === null) {
+    return {
+      ok: false,
+      code: ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
+      message: "the snapshot stream was already settled",
+      details: { indexName: state.expectation.indexName }
+    };
+  }
+  const computed = spool.checksum();
+  if (computed !== state.header.checksum) {
+    return {
+      ok: false,
+      code: ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH,
+      message: "computed checksum does not match header checksum",
+      details: { computed, header: state.header.checksum }
+    };
+  }
+  state.spool = null;
   return {
     ok: true,
-    bytes: state.assembled,
+    snapshot: spool.finish(),
     header: state.header,
     end: state.endPayload
   };
@@ -3081,7 +3247,11 @@ async function fetchSnapshotFromTarget(indexName, partitionId, target, deadline,
     requestId: generateId(),
     payload: encode(requestPayload)
   };
-  const streamState = createSnapshotStreamState({ indexName, partitionId: null });
+  const streamState = createSnapshotStreamState({
+    indexName,
+    partitionId: null,
+    spoolDirectory: deps.spoolDirectory ?? defaultSpoolDirectory()
+  });
   const remainingMs = deadline - Date.now();
   if (remainingMs <= 0) {
     return {
@@ -3097,6 +3267,7 @@ async function fetchSnapshotFromTarget(indexName, partitionId, target, deadline,
     });
     await withDeadline(streamWork, remainingMs, indexName, "stream");
   } catch (err) {
+    abandonSnapshotStream(streamState);
     if (err instanceof NarsilError && err.code === ErrorCodes.SNAPSHOT_SYNC_TIMEOUT) {
       return toStreamFailure(err, ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "snapshot stream deadline exceeded");
     }
@@ -3105,20 +3276,7 @@ async function fetchSnapshotFromTarget(indexName, partitionId, target, deadline,
     }
     return toStreamFailure(err, ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR, "snapshot fetch raised non-transport error");
   }
-  const result = finalizeSnapshotStream(streamState);
-  if (!result.ok) {
-    return result;
-  }
-  const computedChecksum = crc32(result.bytes);
-  if (computedChecksum !== result.header.checksum) {
-    return {
-      ok: false,
-      code: ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH,
-      message: "computed checksum does not match header checksum",
-      details: { computed: computedChecksum, header: result.header.checksum }
-    };
-  }
-  return result;
+  return finalizeSnapshotStream(streamState);
 }
 async function withDeadline(work, ms, indexName, phase) {
   let timer = null;
@@ -3267,9 +3425,9 @@ function surfaceAborted(deps, indexName, primaryNodeId) {
     })
   );
 }
-async function executeEngineRestore(engine, indexName, primaryNodeId, bytes, remainingMs) {
+async function executeEngineRestore(engine, indexName, primaryNodeId, source, remainingMs) {
   try {
-    await withDeadline(engine.restore(indexName, bytes), remainingMs, indexName, "restore");
+    await withDeadline(engine.restore(indexName, source), remainingMs, indexName, "restore");
     return null;
   } catch (err) {
     if (err instanceof NarsilError && err.code === ErrorCodes.SNAPSHOT_SYNC_TIMEOUT) {
@@ -3555,9 +3713,9 @@ function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo,
   }
   return null;
 }
-function createLiveSyncFrameState(indexName, partitionId) {
+function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
   return {
-    snapshotState: createSnapshotStreamState({ indexName, partitionId }),
+    snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
     syncEntries: [],
     sawSnapshotFrame: false,
     error: null
@@ -3659,35 +3817,25 @@ async function applyLiveIncrementalSync(indexName, partitionId, target, entriesP
 async function applyLiveSnapshotSync(state, entry, indexName, partitionId, target, frameState, coordinatorConfig, partitionCount, deadline, deps) {
   const finalized = finalizeSnapshotStream(frameState.snapshotState);
   if (!finalized.ok) {
-    return { ok: false, error: streamFailureToError(finalized) };
+    return { ok: false, error: streamFailureToError({ ...finalized, details: { ...finalized.details, target } }) };
   }
-  const computedChecksum = crc32(finalized.bytes);
-  if (computedChecksum !== finalized.header.checksum) {
-    return {
-      ok: false,
-      error: new NarsilError(
-        ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH,
-        "computed checksum does not match header checksum",
-        {
-          computed: computedChecksum,
-          header: finalized.header.checksum,
-          target
-        }
-      )
-    };
+  let restoreError;
+  try {
+    restoreError = await restoreLiveSnapshotPartition(
+      state,
+      entry,
+      indexName,
+      partitionId,
+      target,
+      finalized.snapshot.read(),
+      coordinatorConfig,
+      partitionCount,
+      deadline,
+      deps
+    );
+  } finally {
+    await finalized.snapshot.discard();
   }
-  const restoreError = await restoreLiveSnapshotPartition(
-    state,
-    entry,
-    indexName,
-    partitionId,
-    target,
-    finalized.bytes,
-    coordinatorConfig,
-    partitionCount,
-    deadline,
-    deps
-  );
   if (restoreError !== null) {
     return { ok: false, error: restoreError };
   }
@@ -3711,7 +3859,7 @@ async function applyLiveSnapshotSync(state, entry, indexName, partitionId, targe
     snapshotHeader: finalized.header
   };
 }
-async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId, primaryNodeId, bytes, coordinatorConfig, partitionCount, deadline, deps) {
+async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId, primaryNodeId, source, coordinatorConfig, partitionCount, deadline, deps) {
   const remainingMs = deadline - Date.now();
   if (remainingMs <= 0) {
     return new NarsilError(
@@ -3735,7 +3883,7 @@ async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId
   }
   try {
     await withDeadline(
-      deps.restoreReplicationPartition(indexName, partitionId, bytes, coordinatorConfig, partitionCount),
+      deps.restoreReplicationPartition(indexName, partitionId, source, coordinatorConfig, partitionCount),
       remainingMs,
       indexName,
       "partition-restore"
@@ -3927,6 +4075,27 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
   return { ok: false, error: lastError };
 }
 async function syncFromTarget(state, entry, indexName, partitionId, target, coordinatorConfig, partitionCount, deadline, deps, abortCheck) {
+  const spoolDirectory = deps.snapshotSpoolDirectory ?? defaultSpoolDirectory();
+  const frameState = createLiveSyncFrameState(indexName, partitionId, spoolDirectory);
+  try {
+    return await syncFrameStateFromTarget(
+      state,
+      entry,
+      indexName,
+      partitionId,
+      target,
+      coordinatorConfig,
+      partitionCount,
+      deadline,
+      deps,
+      abortCheck,
+      frameState
+    );
+  } finally {
+    abandonSnapshotStream(frameState.snapshotState);
+  }
+}
+async function syncFrameStateFromTarget(state, entry, indexName, partitionId, target, coordinatorConfig, partitionCount, deadline, deps, abortCheck, frameState) {
   const logState = readLocalLogState(indexName, partitionId, deps);
   const requestPayload = {
     indexName,
@@ -3940,7 +4109,6 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
     requestId: generateId(),
     payload: encode(requestPayload)
   };
-  const frameState = createLiveSyncFrameState(indexName, partitionId);
   const remainingMs = deadline - Date.now();
   if (remainingMs <= 0) {
     return {
@@ -4172,7 +4340,7 @@ async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
   }
   return configResult.config.schema;
 }
-async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coordinatorSchema, deadline, deps) {
+async function applyRestore(state, entry, indexName, primaryNodeId, source, coordinatorSchema, deadline, deps) {
   const remainingMs = deadline - Date.now();
   if (remainingMs <= 0) {
     surfaceError(
@@ -4191,7 +4359,7 @@ async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coord
     surfaceAborted(deps, indexName, primaryNodeId);
     return false;
   }
-  const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, bytes, remainingMs);
+  const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, source, remainingMs);
   if (restoreError !== null) {
     surfaceError(deps, indexName, primaryNodeId, restoreError);
     return false;
@@ -4312,7 +4480,11 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
     surfaceError(deps, indexName, primaryNodeId, targetsResult);
     return false;
   }
-  const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
+  const fetchDeps = {
+    transport: deps.transport,
+    sourceNodeId: deps.sourceNodeId,
+    spoolDirectory: deps.snapshotSpoolDirectory
+  };
   const fetchResult = await fetchSnapshotFromAnyTarget(
     indexName,
     primaryNodeId,
@@ -4339,20 +4511,25 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
     );
     return false;
   }
-  if (abortCheck()) {
-    surfaceAborted(deps, indexName, primaryNodeId);
-    return false;
+  let restoreSucceeded;
+  try {
+    if (abortCheck()) {
+      surfaceAborted(deps, indexName, primaryNodeId);
+      return false;
+    }
+    restoreSucceeded = await applyRestore(
+      state,
+      entry,
+      indexName,
+      primaryNodeId,
+      fetchResult.snapshot.read(),
+      schemaResult,
+      deadline,
+      deps
+    );
+  } finally {
+    await fetchResult.snapshot.discard();
   }
-  const restoreSucceeded = await applyRestore(
-    state,
-    entry,
-    indexName,
-    primaryNodeId,
-    fetchResult.bytes,
-    schemaResult,
-    deadline,
-    deps
-  );
   if (!restoreSucceeded) {
     return false;
   }
@@ -5216,173 +5393,40 @@ async function applyLocalReplicationEntry(core, entry) {
   }
 }
 
-// src/distribution/cluster-node/local-engine.ts
-async function createClusterLocalEngine(config, hooks) {
-  requireKnownConfig(config);
-  const runnableConfig = await resolveRunnableConfig(config);
-  const core = createEngineCore(runnableConfig, hooks, "partition");
-  await startEngineCore(core, runnableConfig);
-  core.orchestrator.shareMainThread();
-  const engine = createNarsilFromCore(core, runnableConfig);
-  const heldPartitions = createHeldPartitionRecord(core);
-  return Object.assign(engine, {
-    createIndex: (name, indexConfig) => createEngineIndex(core, runnableConfig, name, indexConfig, { adapterBinding: "deferred" }),
-    createIndexWithUuid: (name, indexConfig, indexUuid) => createEngineIndex(core, runnableConfig, name, indexConfig, { indexUuid }),
-    acquireIndexForReplication: (indexName) => core.indexState.acquire(indexName, false),
-    indexUuidOf: (indexName) => core.indexRegistry.get(indexName)?.indexUuid,
-    stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
-    highestPersistedSeqNoOf: (indexName, partitionId) => core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0,
-    heldPartitionsOf: (indexName) => heldPartitions.held(indexName),
-    recordHeldPartition: (indexName, partitionId) => heldPartitions.record(indexName, partitionId),
-    forgetHeldPartition: (indexName, partitionId) => heldPartitions.forget(indexName, partitionId),
-    applyReplicationEntry: (entry) => applyLocalReplicationEntry(core, entry),
-    serializeReplicationPartition: (indexName, partitionId) => serializeReplicationPartition(core, indexName, partitionId),
-    restoreReplicationPartition: (indexName, partitionId, bytes, indexConfig, partitionCount) => restoreReplicationPartition(core, engine, indexName, partitionId, bytes, indexConfig, partitionCount),
-    queryPartitions: (indexName, params, partitionIds, globalStats) => runEngineQuery(core, indexName, params, { partitionIds, globalStats }),
-    preflightPartitions: (indexName, params, partitionIds) => runEnginePreflight(core, indexName, params, { partitionIds }),
-    suggestPartitions: (indexName, params, partitionIds) => runEngineSuggest(core, indexName, params, partitionIds),
-    listPartitions: (indexName, params, partitionIds) => runEngineListDocuments(core, indexName, params, partitionIds),
-    collectQueryStats: (indexName, terms, partitionIds) => runEngineQueryStats(core, indexName, terms, partitionIds),
-    partitionStatsForRead: async (indexName) => {
-      const release = await core.indexState.acquire(indexName);
-      try {
-        return core.requireManager(indexName).getPartitionStats();
-      } finally {
-        release();
-      }
-    }
-  });
-}
-async function stampIndexUuid(core, indexName, indexUuid) {
-  const entry = core.indexRegistry.get(indexName);
-  if (entry === void 0 || entry.indexUuid === indexUuid) {
-    return;
-  }
-  core.indexRegistry.set(indexName, { ...entry, indexUuid });
-  if (core.durability !== null) {
-    await core.durability.manager.persistMetadata(indexName);
-  }
-}
-async function serializeReplicationPartition(core, indexName, partitionId) {
-  core.guardShutdown();
-  const release = await core.indexState.acquire(indexName, false);
-  try {
-    return core.requireManager(indexName).serializePartitionToBytes(partitionId);
-  } finally {
-    release();
-  }
+// src/distribution/cluster-node/replication-index.ts
+function isNestedSchema(value) {
+  return typeof value === "object" && value !== null;
 }
-async function restoreReplicationPartition(core, engine, indexName, partitionId, bytes, indexConfig, partitionCount) {
-  core.guardShutdown();
-  validatePartitionRestoreTarget(indexName, partitionId, partitionCount);
-  const schema = indexConfig.schema;
-  let partition;
-  try {
-    partition = deserializePayloadV2(bytes);
-  } catch (err) {
-    const cause = err instanceof Error ? err.message : String(err);
-    throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `partition snapshot decode failed: ${cause}`, {
-      indexName,
-      partitionId,
-      cause
-    });
-  }
-  validatePartitionSnapshotPayload(indexName, partitionId, partitionCount, schema, partition);
-  const trackPositions = core.indexRegistry.get(indexName)?.config.trackPositions ?? true;
-  createPartitionIndex(partitionId, trackPositions).deserialize(partition, schema);
-  let release = core.indexRegistry.has(indexName) ? await core.indexState.acquire(indexName, false) : null;
-  try {
-    const restoreIndex = await ensureReplicationIndex(
-      core,
-      engine,
-      indexName,
-      indexConfig,
-      partition.language,
-      partitionCount
-    );
-    if (release === null) release = await core.indexState.acquire(indexName, false);
-    if (restoreIndex.created) {
-      await core.orchestrator.replicateToWorkers({
-        type: "createIndex",
-        indexName,
-        config: restoreIndex.config,
-        requestId: `replicate-partition-index-${indexName}`
-      });
+function flattenSchemaInto(schema, prefix, flat) {
+  for (const [field, value] of Object.entries(schema)) {
+    const path = prefix.length === 0 ? field : `${prefix}.${field}`;
+    if (isNestedSchema(value)) {
+      flattenSchemaInto(value, path, flat);
+      continue;
     }
-    const manager = core.requireManager(indexName);
-    manager.deserializePartition(partitionId, partition);
-    await core.orchestrator.replicateToWorkers({
-      type: "deserialize",
-      indexName,
-      partitionId,
-      data: partition,
-      requestId: `replicate-partition-restore-${indexName}-${partitionId}`
-    });
-  } finally {
-    release?.();
+    flat[path] = value;
   }
 }
-function validatePartitionRestoreTarget(indexName, partitionId, partitionCount) {
-  if (!Number.isInteger(partitionId) || partitionId < 0) {
-    throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partitionId must be a non-negative integer", {
-      indexName,
-      partitionId
-    });
-  }
-  if (!Number.isInteger(partitionCount) || partitionCount <= 0 || partitionCount > MAX_PARTITION_COUNT) {
-    throw new NarsilError(
-      ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
-      `partitionCount must be an integer between 1 and ${MAX_PARTITION_COUNT}`,
-      { indexName, partitionCount }
-    );
-  }
-  if (partitionId >= partitionCount) {
-    throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partitionId is outside partitionCount", {
-      indexName,
-      partitionId,
-      partitionCount
-    });
-  }
+function flattenSchema(schema) {
+  const flat = {};
+  flattenSchemaInto(schema, "", flat);
+  return flat;
 }
-function validatePartitionSnapshotPayload(indexName, partitionId, partitionCount, schema, partition) {
-  if (partition.indexName !== indexName) {
-    throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partition snapshot indexName mismatch", {
-      expectedIndexName: indexName,
-      receivedIndexName: partition.indexName
-    });
-  }
-  if (partition.partitionId !== partitionId) {
-    throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partition snapshot partitionId mismatch", {
-      indexName,
-      expectedPartitionId: partitionId,
-      receivedPartitionId: partition.partitionId
-    });
-  }
-  if (partition.totalPartitions !== partitionCount) {
-    throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partition snapshot partition count mismatch", {
-      indexName,
-      partitionId,
-      expectedPartitionCount: partitionCount,
-      receivedPartitionCount: partition.totalPartitions
-    });
-  }
-  const expectedSchema = flattenSchema(schema);
-  for (const [field, expectedType] of Object.entries(expectedSchema)) {
-    if (partition.schema[field] !== expectedType) {
-      throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partition snapshot schema mismatch", {
+function requireSameFlatSchema(indexName, expected, actual, what) {
+  for (const [field, expectedType] of Object.entries(expected)) {
+    if (actual[field] !== expectedType) {
+      throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `${what} schema mismatch`, {
         indexName,
-        partitionId,
         field,
         expected: expectedType,
-        received: partition.schema[field] ?? "(absent)"
+        received: actual[field] ?? "(absent)"
       });
     }
   }
-  for (const field of Object.keys(partition.schema)) {
-    if (!(field in expectedSchema)) {
-      throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "partition snapshot schema contains extra field", {
+  for (const field of Object.keys(actual)) {
+    if (!(field in expected)) {
+      throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `${what} schema contains extra field`, {
         indexName,
-        partitionId,
         field
       });
     }
@@ -5414,7 +5458,12 @@ async function ensureReplicationIndex(core, engine, indexName, coordinatorConfig
       receivedLanguage: language.name
     });
   }
-  validateExistingSchema(indexName, coordinatorConfig.schema, entry.config.schema);
+  requireSameFlatSchema(
+    indexName,
+    flattenSchema(coordinatorConfig.schema),
+    flattenSchema(entry.config.schema),
+    "local index"
+  );
   const manager = core.requireManager(indexName);
   if (manager.partitionCount > partitionCount) {
     throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "local index has more partitions than allocation", {
@@ -5433,45 +5482,222 @@ async function ensureReplicationIndex(core, engine, indexName, coordinatorConfig
   }
   return { config: entry.config, created };
 }
-function validateExistingSchema(indexName, expected, actual) {
-  const expectedSchema = flattenSchema(expected);
-  const actualSchema = flattenSchema(actual);
-  for (const [field, expectedType] of Object.entries(expectedSchema)) {
-    if (actualSchema[field] !== expectedType) {
-      throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "local index schema mismatch", {
-        indexName,
-        field,
-        expected: expectedType,
-        received: actualSchema[field] ?? "(absent)"
+
+// src/distribution/cluster-node/partition-archive.ts
+var VECTOR_FILE_ENVELOPE_VERSION = 2;
+async function archiveReplicationPartition(core, indexName, partitionId, lastSeqNo) {
+  core.guardShutdown();
+  const release = await core.indexState.acquire(indexName, false);
+  try {
+    const entry = core.requireIndex(indexName);
+    const manager = core.requireManager(indexName);
+    let captured = 0;
+    const checkpoint = await buildArchiveFromMemory({
+      indexName,
+      metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
+      manager,
+      vectorIndexes: manager.getVectorIndexes(),
+      appliedSeqNo: () => {
+        captured = lastSeqNo();
+        return captured;
+      },
+      partitionId
+    });
+    const config = await encodeIndexConfiguration(entry);
+    return { stream: archiveStream({ ...checkpoint, config, sourceIndexName: indexName }), lastSeqNo: captured };
+  } finally {
+    release();
+  }
+}
+function restoreFailed(message, details) {
+  throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, message, details);
+}
+function requireRestoreTarget(indexName, partitionId, partitionCount) {
+  if (!Number.isInteger(partitionId) || partitionId < 0) {
+    restoreFailed("partitionId must be a non-negative integer", { indexName, partitionId });
+  }
+  if (!Number.isInteger(partitionCount) || partitionCount <= 0 || partitionCount > MAX_PARTITION_COUNT) {
+    restoreFailed(`partitionCount must be an integer between 1 and ${MAX_PARTITION_COUNT}`, {
+      indexName,
+      partitionCount
+    });
+  }
+  if (partitionId >= partitionCount) {
+    restoreFailed("partitionId is outside partitionCount", { indexName, partitionId, partitionCount });
+  }
+}
+async function receiveArchive(indexName, partitionId, source, schema) {
+  const store = memoryCheckpointStore();
+  let language = "";
+  let received = null;
+  await readSnapshotArchive(createByteReader(source), {
+    config: async (bytes) => {
+      const { payload } = await openChecksummedEnvelope(bytes, {
+        envelopeFormatVersion: 5,
+        what: "index configuration",
+        allowsCompression: true
       });
+      const raw = decode(payload);
+      if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
+      language = raw.language;
+    },
+    manifest: async (manifest, sourceName) => {
+      const listed = manifest.partitions.map((partition) => partition.partitionId);
+      if (listed.length !== 1 || listed[0] !== partitionId) {
+        restoreFailed("partition archive lists other partitions", { indexName, partitionId, listed });
+      }
+      if (manifest.vectors.some((field) => field.graphParts.length > 0)) {
+        restoreFailed("partition archive holds a graph", { indexName, partitionId });
+      }
+      requireSameFlatSchema(indexName, schema, manifest.schema, "partition snapshot");
+      const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
+      received = {
+        ...manifest,
+        partitions: manifest.partitions.map((partition) => ({
+          ...partition,
+          segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
+        })),
+        vectors: manifest.vectors.map((field) => ({
+          ...field,
+          files: field.files.map((file) => ({ ...file, key: rename(file.key) }))
+        }))
+      };
+    },
+    file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
+  });
+  if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
+  return { language, manifest: received, store };
+}
+function emptyPartition(manager, partitionId) {
+  return {
+    indexName: manager.indexName,
+    partitionId,
+    totalPartitions: manager.partitionCount,
+    language: manager.language.name,
+    schema: {},
+    docCount: 0,
+    avgDocLength: 0,
+    documents: {},
+    invertedIndex: {},
+    fieldIndexes: { numeric: {}, boolean: {}, enum: {}, geopoint: {} },
+    statistics: { totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} }
+  };
+}
+async function replacePartition(manager, partitionId, archive) {
+  const vectorIndexes = manager.getVectorIndexes();
+  const held = [...manager.getPartition(partitionId).docIds()];
+  for (const vectorIndex of vectorIndexes.values()) {
+    for (const docId of held) if (vectorIndex.has(docId)) vectorIndex.remove(docId);
+  }
+  manager.deserializePartition(partitionId, emptyPartition(manager, partitionId));
+  const entry = archive.manifest.partitions[0];
+  if (entry !== void 0) {
+    await loadPartitionSegments(archive.store, entry, manager, {
+      schema: archive.manifest.schema,
+      decodesEveryDocument: true
+    });
+  }
+  for (const field of archive.manifest.vectors) {
+    const vectorIndex = vectorIndexes.get(field.fieldPath);
+    if (vectorIndex === void 0) continue;
+    for (const file of field.files) {
+      const bytes = await readListedFile(archive.store, file.key, { bytes: file.bytes, what: "vector file" });
+      const { payload } = await openChecksummedEnvelope(bytes, {
+        envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
+        what: `vector file "${file.key}"`,
+        allowsCompression: false
+      });
+      const decoded = decodeVectorFilePayload(decode(payload));
+      const vectors = bytesToVectors(decoded.vectors);
+      for (let i = 0; i < decoded.docIds.length; i++) {
+        if (isDead(file.dead, i)) continue;
+        const vector = vectors.slice(i * decoded.dimension, (i + 1) * decoded.dimension);
+        vectorIndex.insert(decoded.docIds[i], vector, partitionId);
+      }
     }
+    vectorIndex.scheduleBuild();
   }
-  for (const field of Object.keys(actualSchema)) {
-    if (!(field in expectedSchema)) {
-      throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, "local index schema contains extra field", {
+}
+async function restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount) {
+  core.guardShutdown();
+  requireRestoreTarget(indexName, partitionId, partitionCount);
+  const archive = await receiveArchive(indexName, partitionId, source, flattenSchema(indexConfig.schema));
+  let release = core.indexRegistry.has(indexName) ? await core.indexState.acquire(indexName, false) : null;
+  try {
+    const restoreIndex = await ensureReplicationIndex(
+      core,
+      engine,
+      indexName,
+      indexConfig,
+      archive.language,
+      partitionCount
+    );
+    if (release === null) release = await core.indexState.acquire(indexName, false);
+    if (restoreIndex.created) {
+      await core.orchestrator.replicateToWorkers({
+        type: "createIndex",
         indexName,
-        field
+        config: restoreIndex.config,
+        requestId: `replicate-partition-index-${indexName}`
       });
     }
+    const wasScaledOut = core.orchestrator.desyncIndex(indexName);
+    try {
+      await replacePartition(core.requireManager(indexName), partitionId, archive);
+    } finally {
+      await core.orchestrator.resyncIndex(indexName, wasScaledOut);
+    }
+  } finally {
+    release?.();
   }
 }
-function flattenSchema(schema) {
-  const flat = {};
-  flattenSchemaInto(schema, "", flat);
-  return flat;
-}
-function flattenSchemaInto(schema, prefix, flat) {
-  for (const [field, value] of Object.entries(schema)) {
-    const path = prefix.length === 0 ? field : `${prefix}.${field}`;
-    if (isNestedSchema(value)) {
-      flattenSchemaInto(value, path, flat);
-      continue;
+
+// src/distribution/cluster-node/local-engine.ts
+async function createClusterLocalEngine(config, hooks) {
+  requireKnownConfig(config);
+  const runnableConfig = await resolveRunnableConfig(config);
+  const core = createEngineCore(runnableConfig, hooks, "partition");
+  await startEngineCore(core, runnableConfig);
+  core.orchestrator.shareMainThread();
+  const engine = createNarsilFromCore(core, runnableConfig);
+  const heldPartitions = createHeldPartitionRecord(core);
+  return Object.assign(engine, {
+    createIndex: (name, indexConfig) => createEngineIndex(core, runnableConfig, name, indexConfig, { adapterBinding: "deferred" }),
+    createIndexWithUuid: (name, indexConfig, indexUuid) => createEngineIndex(core, runnableConfig, name, indexConfig, { indexUuid }),
+    acquireIndexForReplication: (indexName) => core.indexState.acquire(indexName, false),
+    indexUuidOf: (indexName) => core.indexRegistry.get(indexName)?.indexUuid,
+    stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
+    highestPersistedSeqNoOf: (indexName, partitionId) => core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0,
+    heldPartitionsOf: (indexName) => heldPartitions.held(indexName),
+    recordHeldPartition: (indexName, partitionId) => heldPartitions.record(indexName, partitionId),
+    forgetHeldPartition: (indexName, partitionId) => heldPartitions.forget(indexName, partitionId),
+    applyReplicationEntry: (entry) => applyLocalReplicationEntry(core, entry),
+    archiveReplicationPartition: (indexName, partitionId, lastSeqNo) => archiveReplicationPartition(core, indexName, partitionId, lastSeqNo),
+    restoreReplicationPartition: (indexName, partitionId, source, indexConfig, partitionCount) => restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount),
+    queryPartitions: (indexName, params, partitionIds, globalStats) => runEngineQuery(core, indexName, params, { partitionIds, globalStats }),
+    preflightPartitions: (indexName, params, partitionIds) => runEnginePreflight(core, indexName, params, { partitionIds }),
+    suggestPartitions: (indexName, params, partitionIds) => runEngineSuggest(core, indexName, params, partitionIds),
+    listPartitions: (indexName, params, partitionIds) => runEngineListDocuments(core, indexName, params, partitionIds),
+    collectQueryStats: (indexName, terms, partitionIds) => runEngineQueryStats(core, indexName, terms, partitionIds),
+    partitionStatsForRead: async (indexName) => {
+      const release = await core.indexState.acquire(indexName);
+      try {
+        return core.requireManager(indexName).getPartitionStats();
+      } finally {
+        release();
+      }
     }
-    flat[path] = value;
-  }
+  });
 }
-function isNestedSchema(value) {
-  return typeof value === "object" && value !== null;
+async function stampIndexUuid(core, indexName, indexUuid) {
+  const entry = core.indexRegistry.get(indexName);
+  if (entry === void 0 || entry.indexUuid === indexUuid) {
+    return;
+  }
+  core.indexRegistry.set(indexName, { ...entry, indexUuid });
+  if (core.durability !== null) {
+    await core.durability.manager.persistMetadata(indexName);
+  }
 }
 
 // src/distribution/cluster-node/local-index-reconcile.ts
@@ -5584,6 +5810,25 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
 }
 
 // src/distribution/cluster-node/snapshot-cache.ts
+function soleSnapshotLease(build) {
+  return { build, release: () => build.discard() };
+}
+function sharedLease(entry, build) {
+  let released = false;
+  return {
+    build,
+    async release() {
+      if (released) return;
+      released = true;
+      entry.holders -= 1;
+      if (entry.holders === 0) await build.discard();
+    }
+  };
+}
+async function joinSnapshotBuild(entry) {
+  entry.holders += 1;
+  return sharedLease(entry, await entry.promise);
+}
 function createSnapshotCacheState(maxConcurrent = DEFAULT_MAX_CONCURRENT_SNAPSHOTS, maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
   if (!Number.isInteger(maxConcurrent) || maxConcurrent <= 0) {
     throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxConcurrent must be a positive integer", {
@@ -5671,7 +5916,7 @@ function releaseStreamSlot(state, handle) {
 async function acquireSnapshotBuild(state, indexName, builder) {
   const existing = state.cache.get(indexName);
   if (existing !== void 0) {
-    return existing.promise;
+    return joinSnapshotBuild(existing);
   }
   if (state.active >= state.maxConcurrent) {
     throw new NarsilError(
@@ -5694,7 +5939,7 @@ async function acquireSnapshotBuild(state, indexName, builder) {
     state.active = Math.max(0, state.active - 1);
   };
   const rawBuild = (async () => builder())();
-  const entry = { promise: rawBuild };
+  const entry = { promise: rawBuild, holders: 0 };
   entry.promise = rawBuild.then(
     (result) => {
       clearEntry(entry);
@@ -5706,7 +5951,7 @@ async function acquireSnapshotBuild(state, indexName, builder) {
     }
   );
   state.cache.set(indexName, entry);
-  return entry.promise;
+  return joinSnapshotBuild(entry);
 }
 var SNAPSHOT_SYNC_ERROR_TYPE = `${ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST}.error`;
 function createSingleResponseSink(respond) {
@@ -5720,8 +5965,7 @@ function createSingleResponseSink(respond) {
   return sink;
 }
 async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, options = {}) {
-  const snapshotBytes = build.bytes;
-  const totalBytes = snapshotBytes.byteLength;
+  const totalBytes = build.totalBytes;
   const checksum = build.checksum;
   const header = {
     lastSeqNo: metadata.lastSeqNo,
@@ -5737,22 +5981,23 @@ async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build
   await yieldToEventLoop();
   let offset = 0;
   let lastYieldAt = now2();
-  while (offset < totalBytes) {
-    const end = Math.min(offset + SNAPSHOT_CHUNK_SIZE, totalBytes);
-    const chunk = snapshotBytes.subarray(offset, end);
-    const chunkPayload = {
-      partitionId: metadata.partitionId,
-      indexName,
-      offset,
-      data: chunk
-    };
-    const chunkBytes = encode(chunkPayload);
-    assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
-    await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
-    offset = end;
-    if (offset < totalBytes && now2() - lastYieldAt >= SNAPSHOT_CHUNK_YIELD_INTERVAL_MS) {
-      await yieldToEventLoop();
-      lastYieldAt = now2();
+  for await (const source of build.read()) {
+    for (let from = 0; from < source.byteLength; from += SNAPSHOT_CHUNK_SIZE) {
+      const chunk = source.subarray(from, Math.min(from + SNAPSHOT_CHUNK_SIZE, source.byteLength));
+      const chunkPayload = {
+        partitionId: metadata.partitionId,
+        indexName,
+        offset,
+        data: chunk
+      };
+      const chunkBytes = encode(chunkPayload);
+      assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
+      await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
+      offset += chunk.byteLength;
+      if (offset < totalBytes && now2() - lastYieldAt >= SNAPSHOT_CHUNK_YIELD_INTERVAL_MS) {
+        await yieldToEventLoop();
+        lastYieldAt = now2();
+      }
     }
   }
   await yieldToEventLoop();
@@ -5833,6 +6078,26 @@ async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
     lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
   };
 }
+async function nodeSnapshotHeaderMetadata(deps, indexName, partitionId) {
+  const fallback = await defaultSnapshotHeaderMetadataProvider(deps.coordinator, indexName);
+  if (partitionId === null) {
+    return fallback;
+  }
+  const localIndex = deps.engine.listIndexes().find((index) => index.name === indexName);
+  const release = localIndex !== void 0 ? await deps.engine.acquireIndexForReplication(indexName) : null;
+  try {
+    const allocation = await deps.coordinator.getAllocation(indexName);
+    const assignment = allocation?.assignments.get(partitionId);
+    const log = deps.getReplicationLog(indexName, partitionId);
+    return {
+      partitionId,
+      primaryTerm: assignment?.primaryTerm ?? fallback.primaryTerm,
+      lastSeqNo: log.newestSeqNo ?? 0
+    };
+  } finally {
+    release?.();
+  }
+}
 async function resolveHeaderMetadata(deps, indexName, partitionId) {
   const provider = deps.resolveHeaderMetadata;
   if (provider === void 0) {
@@ -5862,55 +6127,56 @@ async function acquireAndStream(message, sink, request, deps, options) {
     return;
   }
   try {
-    let build;
+    let lease;
     try {
       const buildForRequest = async () => buildSnapshotForRequest(deps, indexName, options);
-      build = options.disableBuildCache === true ? await buildForRequest() : await acquireSnapshotBuild(deps.state, indexName, buildForRequest);
+      lease = options.disableBuildCache === true ? soleSnapshotLease(await buildForRequest()) : await acquireSnapshotBuild(deps.state, indexName, buildForRequest);
     } catch (err) {
       const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
       const errMessage = err instanceof Error ? err.message : String(err);
       await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
       return;
     }
-    if (build.bytes.byteLength > MAX_SNAPSHOT_SIZE_BYTES) {
-      await respondError(
-        sink,
-        deps.nodeId,
-        message.requestId,
-        ErrorCodes.SNAPSHOT_SYNC_TOO_LARGE,
-        `Snapshot size ${build.bytes.byteLength} exceeds the ${MAX_SNAPSHOT_SIZE_BYTES} byte limit`
-      );
-      return;
-    }
-    let streamHandle;
-    try {
-      streamHandle = acquireStreamSlot(deps.state, indexName);
-    } catch (err) {
-      const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
-      const errMessage = err instanceof Error ? err.message : String(err);
-      await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
-      return;
-    }
     try {
-      const metadata = options.metadata ?? await resolveHeaderMetadata(deps, indexName, request.partitionId ?? null);
-      await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata, {
-        closeOnEnd: options.closeOnEnd
-      });
-      await options.afterSnapshot?.(sink);
-      if (options.closeOnEnd === false) {
-        sink.closed = true;
-      }
+      await streamLeasedSnapshot(message, sink, request, deps, options, lease.build);
     } finally {
-      releaseStreamSlot(deps.state, streamHandle);
+      await lease.release();
     }
   } finally {
     releaseSourceSlot(deps.state, sourceHandle);
   }
 }
+async function streamLeasedSnapshot(message, sink, request, deps, options, build) {
+  const { indexName } = request;
+  let streamHandle;
+  try {
+    streamHandle = acquireStreamSlot(deps.state, indexName);
+  } catch (err) {
+    const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
+    const errMessage = err instanceof Error ? err.message : String(err);
+    await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
+    return;
+  }
+  try {
+    const resolved = options.metadata ?? await resolveHeaderMetadata(deps, indexName, request.partitionId ?? null);
+    const metadata = build.lastSeqNo === void 0 ? resolved : { ...resolved, lastSeqNo: build.lastSeqNo };
+    await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata, {
+      closeOnEnd: options.closeOnEnd
+    });
+    await options.afterSnapshot?.(sink, build);
+    if (options.closeOnEnd === false) {
+      sink.closed = true;
+    }
+  } finally {
+    releaseStreamSlot(deps.state, streamHandle);
+  }
+}
 async function buildSnapshot(deps, indexName) {
-  let bytes;
   try {
-    bytes = await deps.engine.snapshot(indexName);
+    return await spoolSnapshotStream(
+      deps.spoolDirectory ?? defaultSpoolDirectory(),
+      await deps.engine.snapshot(indexName)
+    );
   } catch (err) {
     if (err instanceof NarsilError) {
       throw err;
@@ -5921,8 +6187,6 @@ async function buildSnapshot(deps, indexName) {
       cause: errMessage
     });
   }
-  const checksum = bytes.byteLength <= MAX_SNAPSHOT_SIZE_BYTES ? crc32(bytes) : 0;
-  return { bytes, checksum };
 }
 function buildSnapshotForRequest(deps, indexName, options) {
   if (options.buildSnapshot !== void 0) {
@@ -6131,7 +6395,6 @@ var MAX_FACET_VALUE_LENGTH = 1024;
 var MAX_GROUPS_PER_RESPONSE = 65536;
 var MAX_FACET_SIZE = 1e3;
 var DEFAULT_MAX_FACET_BUCKETS = 100;
-var MAX_VECTOR_DIMENSION = 8192;
 var MAX_VECTOR_TEXT_LENGTH = 16384;
 
 // src/distribution/query/validators/common.ts
@@ -6867,11 +7130,11 @@ function validateVectorParams(value) {
       );
     }
     const dims = value.value.length;
-    if (dims === 0 || dims > MAX_VECTOR_DIMENSION) {
+    if (dims === 0 || dims > MAX_STORED_VECTOR_DIMENSION) {
       throwInvalid(
         CONFIG_INVALID,
-        `Invalid SearchPayload: "params.vector.value" length must be between 1 and ${MAX_VECTOR_DIMENSION}`,
-        { length: dims, limit: MAX_VECTOR_DIMENSION }
+        `Invalid SearchPayload: "params.vector.value" length must be between 1 and ${MAX_STORED_VECTOR_DIMENSION}`,
+        { length: dims, limit: MAX_STORED_VECTOR_DIMENSION }
       );
     }
     for (let i = 0; i < dims; i++) {
@@ -7062,6 +7325,9 @@ function validateParams(params) {
   if (params.prefix !== null) {
     validateBooleanParam(params.prefix, "prefix");
   }
+  if (params.partialWords !== null && params.partialWords !== void 0) {
+    validateBooleanParam(params.partialWords, "partialWords");
+  }
   if (params.exact !== null) {
     validateBooleanParam(params.exact, "exact");
   }
@@ -7671,6 +7937,7 @@ function wireParamsToLocal(wire, facetShardSize) {
     termMatch: wire.termMatch ?? void 0,
     prefixLength: wire.prefixLength ?? void 0,
     prefix: wire.prefix ?? void 0,
+    partialWords: wire.partialWords ?? void 0,
     exact: wire.exact ?? void 0,
     pinned: wire.pinned !== null ? wire.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : void 0,
     mode: wire.mode ?? void 0,
@@ -7766,6 +8033,7 @@ function localParamsToWire(params) {
     termMatch: params.termMatch ?? null,
     prefixLength: params.prefixLength ?? null,
     prefix: params.prefix ?? null,
+    partialWords: params.partialWords ?? null,
     exact: params.exact ?? null,
     pinned: params.pinned !== void 0 ? params.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : null,
     mode: params.mode ?? null,
@@ -8073,6 +8341,8 @@ function toWireSortValue2(value) {
   if (typeof value === "boolean") return value;
   return null;
 }
+
+// src/distribution/replication/sync-primary.ts
 function decideSyncTier(log, replicaLastSeqNo) {
   const oldest = log.oldestSeqNo;
   if (oldest === void 0) {
@@ -8194,7 +8464,8 @@ async function handleSyncRequestMessage(message, respond, deps) {
         engine: deps.engine,
         coordinator: deps.coordinator,
         state: deps.snapshotSyncState,
-        resolveHeaderMetadata: deps.resolveHeaderMetadata
+        resolveHeaderMetadata: deps.resolveHeaderMetadata,
+        spoolDirectory: deps.snapshotSpoolDirectory
       },
       {
         metadata: {
@@ -8205,11 +8476,16 @@ async function handleSyncRequestMessage(message, respond, deps) {
         closeOnEnd: false,
         disableBuildCache: true,
         buildSnapshot: async () => {
-          const bytes = await deps.engine.serializeReplicationPartition(request.indexName, request.partitionId);
-          return { bytes, checksum: crc32(bytes) };
+          const archive = await deps.engine.archiveReplicationPartition(
+            request.indexName,
+            request.partitionId,
+            () => log.localLogEnd
+          );
+          const directory = deps.snapshotSpoolDirectory ?? defaultSpoolDirectory();
+          return { ...await spoolSnapshotStream(directory, archive.stream), lastSeqNo: archive.lastSeqNo };
         },
-        afterSnapshot: async (trailingSink) => {
-          const entries = log.getEntriesFrom(snapshotSeqNo + 1);
+        afterSnapshot: async (trailingSink, build) => {
+          const entries = log.getEntriesFrom((build.lastSeqNo ?? snapshotSeqNo) + 1);
           await sendSyncEntriesResponse(message, trailingSink, deps, entries);
         }
       }
@@ -8325,7 +8601,7 @@ async function forwardInsertToRemote(indexName, document, docId, primaryNodeId,
     indexName,
     documentId: docId,
     operation: "insert",
-    document: encode(document),
+    document: encodeDocument(document),
     updateFields: null
   };
   const message = createForwardMessage(payload, deps.nodeId);
@@ -8345,7 +8621,7 @@ async function forwardUpdateToRemote(indexName, document, docId, primaryNodeId,
     indexName,
     documentId: docId,
     operation: "update",
-    document: encode(document),
+    document: encodeDocument(document),
     updateFields: null
   };
   const message = createForwardMessage(payload, deps.nodeId);
@@ -8371,6 +8647,8 @@ async function sendToNode(nodeId, message, deps) {
     details: { targetNodeId: nodeId }
   });
 }
+
+// src/distribution/cluster-node/write-routing/replication.ts
 async function assertPrimaryWriteAuthority(entry, deps) {
   const table = await deps.coordinator.getAllocation(entry.indexName);
   const currentAssignment = table?.assignments.get(entry.partitionId);
@@ -8400,7 +8678,7 @@ function appendIndexReplicationEntry(indexName, partitionId, assignment, documen
     partitionId,
     indexName,
     documentId,
-    document: encode(document)
+    document: encodeDocument(document)
   });
 }
 function appendDeleteReplicationEntry(indexName, partitionId, assignment, documentId, deps) {
@@ -8884,7 +9162,7 @@ function toWireOperation(item) {
   return {
     documentId: item.documentId,
     operation: item.operation,
-    document: item.document === null ? null : encode(item.document),
+    document: item.document === null ? null : encodeDocument(item.document),
     updateFields: null
   };
 }
@@ -9291,9 +9569,15 @@ function validateCreateIndexOptions(partitionCount, replicationFactor) {
 async function routeCreateIndex(name, requestedConfig, options, coordinator, engine) {
   validateIndexName(name);
   const config = { ...requestedConfig, schema: validateSchema(requestedConfig.schema) };
+  requireCreatableSchema(config.schema);
+  requireSurfaceFormsForPartialFields(config.schema, config.surfaceForms);
   const partitionCount = options?.partitionCount ?? DEFAULT_PARTITION_COUNT;
   const replicationFactor = options?.replicationFactor ?? DEFAULT_REPLICATION_FACTOR;
   validateCreateIndexOptions(partitionCount, replicationFactor);
+  requireConfigurationWithinSizeLimit(
+    withPartitionCount(config, partitionCount),
+    getLanguage(config.language ?? "english")
+  );
   const constraints = {
     zoneAwareness: false,
     zoneAttribute: "zone",
@@ -9574,7 +9858,8 @@ function createDataNodeHandler(deps) {
         engine: deps.engine,
         coordinator: deps.coordinator,
         state: deps.snapshotSyncState,
-        resolveHeaderMetadata: deps.resolveHeaderMetadata
+        resolveHeaderMetadata: deps.resolveHeaderMetadata,
+        spoolDirectory: deps.snapshotSpoolDirectory
       });
       return;
     }
@@ -11529,6 +11814,10 @@ async function createClusterNode(config) {
     onIndexClose: (indexName) => deleteIndexReplicationLogs(replicationLogs, indexName)
   });
   localEngine = engine;
+  const snapshotSpool = await openSnapshotSpoolDirectory(config.engine?.durability?.directory).catch(async (err) => {
+    await engine.shutdown();
+    throw err;
+  });
   function replicationLogKey2(indexName, partitionId) {
     return replicationLogKey(indexName, partitionId);
   }
@@ -11588,7 +11877,8 @@ async function createClusterNode(config) {
     resetReplicationLog: seedReplicationLog2,
     applyReplicationEntry: engine.applyReplicationEntry,
     restoreReplicationPartition: engine.restoreReplicationPartition,
-    onError: forwardOnError
+    onError: forwardOnError,
+    snapshotSpoolDirectory: snapshotSpool.path
   };
   const primaryPartitionDeps = {
     engine,
@@ -11720,6 +12010,7 @@ async function createClusterNode(config) {
           coordinator: config.coordinator,
           writeDeps,
           snapshotSyncState: snapshotSyncHandlerState,
+          snapshotSpoolDirectory: snapshotSpool.path,
           resolveHeaderMetadata: resolveSnapshotHeaderMetadata,
           isBootstrapSynced: (indexName, partitionId) => hasCompletedBootstrapSync(bootstrapSyncState, indexName, partitionId)
         });
@@ -11751,28 +12042,16 @@ async function createClusterNode(config) {
       }
       await Promise.all([...heldPartitionWrites]);
       await engine.shutdown();
+      await snapshotSpool.dispose();
     }
   };
   return node;
-  async function resolveSnapshotHeaderMetadata(indexName, partitionId) {
-    const fallback = await defaultSnapshotHeaderMetadataProvider(config.coordinator, indexName);
-    if (partitionId === null) {
-      return fallback;
-    }
-    const localIndex = engine.listIndexes().find((index) => index.name === indexName);
-    const release = localIndex !== void 0 ? await engine.acquireIndexForReplication(indexName) : null;
-    try {
-      const allocation = await config.coordinator.getAllocation(indexName);
-      const assignment = allocation?.assignments.get(partitionId);
-      const log = getReplicationLog2(indexName, partitionId);
-      return {
-        partitionId,
-        primaryTerm: assignment?.primaryTerm ?? fallback.primaryTerm,
-        lastSeqNo: log.newestSeqNo ?? 0
-      };
-    } finally {
-      release?.();
-    }
+  function resolveSnapshotHeaderMetadata(indexName, partitionId) {
+    return nodeSnapshotHeaderMetadata(
+      { coordinator: config.coordinator, engine, getReplicationLog: getReplicationLog2 },
+      indexName,
+      partitionId
+    );
   }
 }
 
@@ -11875,6 +12154,8 @@ function clusterNodeEngine(node, options) {
     rebuildAnalysis: () => unsupported("rebuildAnalysis"),
     snapshot: () => unsupported("snapshot"),
     restore: () => unsupported("restore"),
+    exportDocuments: () => unsupported("exportDocuments"),
+    importDocumentExport: () => unsupported("importDocumentExport"),
     rebalance: () => unsupported("rebalance"),
     updatePartitionConfig: () => unsupported("updatePartitionConfig"),
     compactVectors: () => unsupported("compactVectors"),