@delali/narsil 0.3.1-next.72 → 0.3.1-next.74

package/dist/{chunk-EDMCZLSJ.mjs → chunk-NOZUQM7J.mjs}

@@ -1,6 +1,9 @@
-import { MAX_SORT_FIELDS } from './chunk-M6XF5HDE.mjs';
-import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-P2ZZN32C.mjs';
-import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-RV77FR2F.mjs';
+import { MAX_SORT_FIELDS } from './chunk-3DDUNI6N.mjs';
+import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-2E35UM3G.mjs';
+import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
+import { mkdtemp, open, rm } from 'fs/promises';
+import { tmpdir } from 'os';
+import path from 'path';
 
 // src/server/capabilities.ts
 var ASYNC_IMPORT_CAPABILITY = "documents.import.async";
@@ -8,12 +11,14 @@ var TASK_CANCEL_CAPABILITY = "tasks.cancel";
 var TASK_FILTER_CAPABILITY = "tasks.filter";
 var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
 var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
+var DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
 var SERVER_CAPABILITIES = [
   ASYNC_IMPORT_CAPABILITY,
   TASK_CANCEL_CAPABILITY,
   TASK_FILTER_CAPABILITY,
   REBUILD_ANALYSIS_CAPABILITY,
-  INDEX_LIFECYCLE_CAPABILITY
+  INDEX_LIFECYCLE_CAPABILITY,
+  DOCUMENT_EXPORT_CAPABILITY
 ];
 
 // src/server/concurrency-gate.ts
@@ -171,7 +176,6 @@ var STATUS_BY_CODE = {
   SNAPSHOT_SYNC_HEADER_INVALID: 400,
   SNAPSHOT_SYNC_FRAME_INVALID: 400,
   SNAPSHOT_SYNC_INDEX_NOT_FOUND: 404,
-  SNAPSHOT_SYNC_TOO_LARGE: 413,
   SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED: 413,
   SNAPSHOT_SYNC_ABORTED: 409,
   SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: 503,
@@ -226,6 +230,11 @@ function encodeJson(value) {
 var BYTES_PER_MB = 1024 * 1024;
 var DEFAULT_MAX_BODY_BYTES = 16 * BYTES_PER_MB;
 var DEFAULT_MAX_IMPORT_BYTES = 100 * BYTES_PER_MB;
+var DEFAULT_MAX_RESTORE_BYTES = 64 * 1024 * BYTES_PER_MB;
+var SPOOL_PAUSE_ABOVE_BYTES = 8 * BYTES_PER_MB;
+var SNAPSHOT_ARCHIVE_CONTENT_TYPE = "application/x-tar";
+var DOCUMENT_EXPORT_CONTENT_TYPE = "application/x-ndjson";
+var RELAY_STREAM_CHUNKS_IN_FLIGHT = 4;
 var DEFAULT_MAX_LINE_BYTES = 4 * BYTES_PER_MB;
 var DEFAULT_IMPORT_BATCH_SIZE = 1e3;
 var IMPORT_FLUSHES_IN_FLIGHT = 4;
@@ -300,8 +309,56 @@ function sendError(res, status, code, message, details) {
     res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
   });
 }
-function sendBinary(res, data, abort, status = 200) {
-  streamBody(res, abort, data, status, [["Content-Type", "application/octet-stream"]]);
+async function sendStream(res, abort, source, headers) {
+  const reader = source.getReader();
+  let wake = null;
+  abort.onAbort(() => wake?.());
+  const waitUntilWritable = () => new Promise((resolve) => {
+    wake = () => {
+      wake = null;
+      resolve();
+    };
+    res.onWritable(() => {
+      wake?.();
+      return true;
+    });
+  });
+  let started = false;
+  const begin = () => {
+    if (started) return;
+    started = true;
+    res.writeStatus(statusLine(200));
+    for (const [key, value] of headers) res.writeHeader(key, value);
+  };
+  try {
+    for (; ; ) {
+      const step = await reader.read();
+      if (abort.aborted) {
+        await reader.cancel().catch(() => void 0);
+        return;
+      }
+      if (step.done) {
+        res.cork(() => {
+          begin();
+          res.end();
+        });
+        return;
+      }
+      let writable = true;
+      res.cork(() => {
+        begin();
+        writable = res.write(step.value);
+      });
+      if (!writable && !abort.aborted) await waitUntilWritable();
+    }
+  } catch (err) {
+    await reader.cancel().catch(() => void 0);
+    if (abort.aborted) return;
+    if (!started) throw err;
+    res.cork(() => {
+      res.close();
+    });
+  }
 }
 function sendRelayed(res, abort, status, headers, body) {
   if (abort.aborted) return;
@@ -496,7 +553,14 @@ function validateRebalance(body) {
   }
   return null;
 }
-var TASK_TYPES = ["optimizeVectors", "rebalance", "restore", "import", "rebuildAnalysis"];
+var TASK_TYPES = [
+  "optimizeVectors",
+  "rebalance",
+  "restore",
+  "import",
+  "importExport",
+  "rebuildAnalysis"
+];
 var TASK_STATUSES = ["running", "succeeded", "failed", "cancelled"];
 function parseCountParam(raw, field, minimum) {
   if (raw === null) return minimum;
@@ -631,6 +695,87 @@ function createVersionHandler(deps) {
   }
   return { report };
 }
+var NotAdmittedError = class extends Error {
+};
+async function removeDirectory(directory) {
+  if (directory !== null) await rm(directory, { recursive: true, force: true });
+}
+function spoolBody(res, maxBytes, abort, admitted) {
+  return new Promise((resolve, reject) => {
+    const state = {
+      directory: null,
+      handle: null,
+      chain: Promise.resolve(),
+      queued: 0,
+      total: 0,
+      paused: true,
+      done: false
+    };
+    const fail = (error, answer) => {
+      if (state.done) return;
+      state.done = true;
+      if (answer && !abort.aborted) {
+        sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "The server could not store the request body");
+      }
+      void state.chain.catch(() => void 0).then(async () => {
+        await state.handle?.close().catch(() => void 0);
+        await removeDirectory(state.directory).catch(() => void 0);
+      }).finally(() => reject(error));
+    };
+    const failFromChain = (error) => {
+      if (error instanceof NotAdmittedError) fail(error, false);
+      else fail(error instanceof Error ? error : new Error(String(error)), true);
+    };
+    res.pause();
+    state.chain = (async () => {
+      if (!await admitted) throw new NotAdmittedError("the request was not admitted");
+      state.directory = await mkdtemp(path.join(tmpdir(), "narsil-spool-"));
+      state.handle = await open(path.join(state.directory, "body"), "w");
+      if (!abort.aborted && !state.done && state.queued <= SPOOL_PAUSE_ABOVE_BYTES) {
+        state.paused = false;
+        res.resume();
+      }
+    })();
+    state.chain.catch(failFromChain);
+    abort.onAbort(() => fail(new Error("aborted"), false));
+    res.onData((chunk, isLast) => {
+      if (state.done || abort.aborted) return;
+      state.total += chunk.byteLength;
+      if (maxBytes > 0 && state.total > maxBytes) {
+        sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
+        fail(new Error("payload too large"), false);
+        return;
+      }
+      const copy = new Uint8Array(chunk.slice(0));
+      state.queued += copy.byteLength;
+      if (!state.paused && state.queued > SPOOL_PAUSE_ABOVE_BYTES) {
+        state.paused = true;
+        res.pause();
+      }
+      state.chain = state.chain.then(async () => {
+        if (state.handle === null) throw new Error("the spool file is not open");
+        await state.handle.write(copy);
+        state.queued -= copy.byteLength;
+        if (state.paused && state.queued <= SPOOL_PAUSE_ABOVE_BYTES && !abort.aborted && !state.done) {
+          state.paused = false;
+          res.resume();
+        }
+      });
+      state.chain.catch(failFromChain);
+      if (!isLast) return;
+      state.chain.then(async () => {
+        if (state.done) return;
+        state.done = true;
+        await state.handle?.close();
+        resolve({
+          path: path.join(state.directory ?? "", "body"),
+          bytes: state.total,
+          remove: () => removeDirectory(state.directory)
+        });
+      }, failFromChain);
+    });
+  });
+}
 
 // src/server/request.ts
 function initAbortHandler(res) {
@@ -732,6 +877,19 @@ function sinkWritingAfterStatus(res, headers) {
       res.end(body);
       return sink;
     },
+    write(chunk) {
+      return res.write(chunk);
+    },
+    pause() {
+      res.pause();
+    },
+    resume() {
+      res.resume();
+    },
+    close() {
+      res.close();
+      return sink;
+    },
     endWithoutBody() {
       res.endWithoutBody();
       return sink;
@@ -830,18 +988,33 @@ function createRouteRunner(deps) {
       }
       const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
       const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
-      Promise.all([bodyPromise, hookPromise]).then(async ([rawBody, authorization]) => {
-        if (abort.aborted) return;
+      const spoolPromise = opts.spoolsBody === true ? spoolBody(
+        res,
+        opts.maxBytes,
+        abort,
+        hookPromise.then((authorization) => authorization.allowed)
+      ).catch(() => null) : Promise.resolve(null);
+      Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
+        const discardSpool = () => {
+          void spooledBody?.remove().catch(() => void 0);
+        };
+        if (abort.aborted) {
+          discardSpool();
+          return;
+        }
         if (!authorization.allowed) {
+          discardSpool();
           sendDenial(res, authorization);
           return;
         }
+        if (opts.spoolsBody === true && spooledBody === null) return;
         if (gated && !gate.tryAcquire()) {
+          discardSpool();
           sendError(res, 429, ServerErrorCodes.TOO_MANY_REQUESTS, "The server is at capacity; retry shortly");
           return;
         }
         try {
-          await handler({ res, params, query, contentType, rawBody, abort, hookContext });
+          await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
         } catch (err) {
           if (!abort.aborted) {
             sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
@@ -916,7 +1089,14 @@ function serverRoutes(limits) {
       method: "post",
       path: "/indexes/:name/restore",
       handler: "restore",
-      options: { paramCount: 1, needsBody: true, maxBytes: maxImportBytes }
+      options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
+    },
+    { method: "get", path: "/indexes/:name/export", handler: "exportDocuments", options: named },
+    {
+      method: "post",
+      path: "/indexes/:name/import",
+      handler: "importExport",
+      options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
     }
   ];
 }
@@ -975,4 +1155,4 @@ function childAppOf(app) {
   return candidate;
 }
 
-export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendBinary, sendError, sendRelayed, serializeBatchResult, serializeNarsilError, serverRoutes, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
+export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };

package/dist/{chunk-LCCL6EEH.mjs → chunk-PO53FV4Q.mjs}

@@ -1,7 +1,8 @@
 import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
-import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-P2ZZN32C.mjs';
+import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-2E35UM3G.mjs';
+import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
 import { compareCodePoints } from './chunk-LJYTW7GT.mjs';
-import { NarsilError, ErrorCodes } from './chunk-RV77FR2F.mjs';
+import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
 import { readSync, closeSync, openSync } from 'fs';
 
 // src/vector/constants.ts
@@ -3651,6 +3652,13 @@ function createVectorStore(options) {
         yield [docId, entryAt(open, ordinal)];
       }
     },
+    *docIds() {
+      if (open === null) return;
+      for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
+        const docId = ordToDoc[ordinal];
+        if (docId !== void 0) yield docId;
+      }
+    },
     clear() {
       forgetDocuments();
       if (open === null) return;
@@ -3789,30 +3797,24 @@ function savedTopLayer(state, ordinal) {
   if (ordinal === NO_ORDINAL || isTombstoned(state, ordinal)) return NO_LAYER;
   return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
 }
-function largestNeighbourValueCount(state, topLayers) {
+function neighbourValueCount(state, topLayers, start, end) {
   let values = 0;
-  for (let number = 0; number < topLayers.length; number++) {
+  for (let number = start; number < end; number++) {
     const top = topLayers[number];
     if (top < 0) continue;
     values += state.adjacency.level0Stride + top * state.adjacency.upperStride;
   }
   return values;
 }
-function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
+function serializePart(state, numbers, start, end) {
+  const { numberOfOrdinal, ordinalOfNumber, topLayers } = numbers;
   const adjacency = state.adjacency;
-  const slots = adjacencySlots(adjacency);
-  const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
-  const levels = new Uint8Array(ordinalOfNumber.length);
-  for (let number = 0; number < ordinalOfNumber.length; number++) {
-    const ordinal = ordinalOfNumber[number];
-    if (ordinal >= slots) continue;
-    topLayers[number] = savedTopLayer(state, ordinal);
-    levels[number] = topLayers[number] + 1;
-  }
-  const values = new Uint32Array(largestNeighbourValueCount(state, topLayers));
+  const levels = new Uint8Array(end - start);
+  const values = new Uint32Array(neighbourValueCount(state, topLayers, start, end));
   let cursor = 0;
-  for (let number = 0; number < ordinalOfNumber.length; number++) {
+  for (let number = start; number < end; number++) {
     const top = topLayers[number];
+    levels[number - start] = top + 1;
     for (let layer = 0; layer <= top; layer++) {
       const base = layerBase(adjacency, ordinalOfNumber[number], layer);
       const array = layerArray(adjacency, layer);
@@ -3830,18 +3832,33 @@ function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
       values[countAt] = cursor - countAt - 1;
     }
   }
+  return { levels, neighbours: littleEndianBytesOf(values.subarray(0, cursor)) };
+}
+function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize) {
+  const slots = adjacencySlots(state.adjacency);
+  const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
+  for (let number = 0; number < ordinalOfNumber.length; number++) {
+    const ordinal = ordinalOfNumber[number];
+    if (ordinal >= slots) continue;
+    topLayers[number] = savedTopLayer(state, ordinal);
+  }
   const entryOrdinal = entryPointOf(state);
   const entryNumber = entryOrdinal >= 0 && entryOrdinal < numberOfOrdinal.length ? numberOfOrdinal[entryOrdinal] : NO_NUMBER;
   const entryHoldsNode = entryNumber !== NO_NUMBER && topLayers[entryNumber] >= 0;
-  return {
+  const header = {
     entryPoint: entryHoldsNode ? entryNumber : null,
     maxLayer: Math.max(0, Math.min(topLayerOf(state), MAX_LAYER_CAP)),
     m: state.M,
     efConstruction: state.efCons,
-    metric: state.buildMetric,
-    levels,
-    neighbours: littleEndianBytesOf(values.subarray(0, cursor))
+    metric: state.buildMetric
   };
+  const numbers = { numberOfOrdinal, ordinalOfNumber, topLayers };
+  const parts = [];
+  for (let start = 0; start < ordinalOfNumber.length; start += partSize) {
+    const end = Math.min(start + partSize, ordinalOfNumber.length);
+    parts.push({ ...header, ...serializePart(state, numbers, start, end) });
+  }
+  return parts;
 }
 function endsInsideAList() {
   throw new NarsilError(
@@ -3862,95 +3879,78 @@ function requireWholeNeighbourLists(graph) {
     }
   }
 }
-function topLayerIn(graph, number) {
-  return number < graph.levels.length ? graph.levels[number] - 1 : -1;
+function topLayerInParts(graph, number) {
+  const part = graph.parts[Math.floor(number / graph.partSize)];
+  if (part === void 0) return -1;
+  const at = number % graph.partSize;
+  return at < part.levels.length ? part.levels[at] - 1 : -1;
+}
+function highestNodeOrdinal(graph, ordinalAt) {
+  let highest = -1;
+  for (let p = 0; p < graph.parts.length; p++) {
+    const levels = graph.parts[p].levels;
+    for (let at = 0; at < levels.length; at++) {
+      const ordinal = ordinalAt(p * graph.partSize + at);
+      if (levels[at] > 0 && ordinal > highest) highest = ordinal;
+    }
+  }
+  return highest;
 }
 function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
-  requireWholeNeighbourLists(graph);
+  for (const part of graph.parts) requireWholeNeighbourLists(part);
   resetGraph(state);
-  const values = valuesOf(graph.neighbours);
   const ordinalAt = (number) => number < ordinalOfNumber.length ? ordinalOfNumber[number] : NO_ORDINAL;
-  let highestOrdinal = -1;
-  for (let number = 0; number < graph.levels.length; number++) {
-    if (graph.levels[number] > 0 && ordinalAt(number) > highestOrdinal) highestOrdinal = ordinalAt(number);
-  }
+  const highestOrdinal = highestNodeOrdinal(graph, ordinalAt);
   if (highestOrdinal >= 0) ensureCapacity(state, highestOrdinal + 1);
   const kept = [];
-  let cursor = 0;
   let highestLayer = -1;
   let nodeOnHighestLayer = NO_ORDINAL;
-  for (let number = 0; number < graph.levels.length; number++) {
-    const top = graph.levels[number] - 1;
-    if (top < 0) continue;
-    const ordinal = ordinalAt(number);
-    const keptTop = Math.min(top, MAX_LAYER_CAP);
-    if (ordinal !== NO_ORDINAL) {
-      createNode(state.adjacency, ordinal, keptTop);
-      Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
-      if (keptTop > highestLayer) {
-        highestLayer = keptTop;
-        nodeOnHighestLayer = ordinal;
-      }
-    }
-    for (let layer = 0; layer <= top; layer++) {
-      const count = values[cursor];
-      kept.length = 0;
-      const limit = maxConns(state, layer);
-      for (let i = 1; i <= count && kept.length < limit; i++) {
-        const neighbour = values[cursor + i];
-        const neighbourOrdinal = ordinalAt(neighbour);
-        if (neighbourOrdinal === NO_ORDINAL || neighbour === number || topLayerIn(graph, neighbour) < layer) continue;
-        if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
+  for (let p = 0; p < graph.parts.length; p++) {
+    const part = graph.parts[p];
+    const values = valuesOf(part.neighbours);
+    let cursor = 0;
+    for (let at = 0; at < part.levels.length; at++) {
+      const number = p * graph.partSize + at;
+      const top = part.levels[at] - 1;
+      if (top < 0) continue;
+      const ordinal = ordinalAt(number);
+      const keptTop = Math.min(top, MAX_LAYER_CAP);
+      if (ordinal !== NO_ORDINAL) {
+        createNode(state.adjacency, ordinal, keptTop);
+        Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
+        if (keptTop > highestLayer) {
+          highestLayer = keptTop;
+          nodeOnHighestLayer = ordinal;
+        }
       }
-      cursor += count + 1;
-      if (ordinal !== NO_ORDINAL && layer <= keptTop) {
-        replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
+      for (let layer = 0; layer <= top; layer++) {
+        const count = values[cursor];
+        kept.length = 0;
+        const limit = maxConns(state, layer);
+        for (let i = 1; i <= count && kept.length < limit; i++) {
+          const neighbour = values[cursor + i];
+          const neighbourOrdinal = ordinalAt(neighbour);
+          if (neighbourOrdinal === NO_ORDINAL || neighbour === number) continue;
+          if (topLayerInParts(graph, neighbour) < layer) continue;
+          if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
+        }
+        cursor += count + 1;
+        if (ordinal !== NO_ORDINAL && layer <= keptTop) {
+          replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
+        }
       }
     }
   }
-  const entryOrdinal = graph.entryPoint === null ? NO_ORDINAL : ordinalAt(graph.entryPoint);
-  const entryTop = graph.entryPoint === null ? -1 : Math.min(topLayerIn(graph, graph.entryPoint), MAX_LAYER_CAP);
+  const first = graph.parts[0];
+  const entryPoint = first === void 0 ? null : first.entryPoint;
+  const entryOrdinal = entryPoint === null ? NO_ORDINAL : ordinalAt(entryPoint);
+  const entryTop = entryPoint === null ? -1 : Math.min(topLayerInParts(graph, entryPoint), MAX_LAYER_CAP);
   const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
   Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
   Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
 }
 
 // src/vector/hnsw/persistence.ts
-function serializeGraph(state, store) {
-  const nodeArray = [];
-  const slots = adjacencySlots(state.adjacency);
-  for (let ord = 0; ord < slots; ord++) {
-    const maxLayer = nodeMaxLayer(state, ord);
-    if (maxLayer === -1) continue;
-    if (isTombstoned(state, ord)) continue;
-    const docId = store.docIdForOrdinal(ord);
-    if (docId === void 0) continue;
-    const layerConns = [];
-    for (let layer = 0; layer <= maxLayer; layer++) {
-      const liveNeighbors = [];
-      for (const neighborOrd of collectNeighbors(state.adjacency, ord, layer)) {
-        if (isTombstoned(state, neighborOrd)) continue;
-        const neighborDoc = store.docIdForOrdinal(neighborOrd);
-        if (neighborDoc === void 0) continue;
-        liveNeighbors.push(neighborDoc);
-      }
-      if (liveNeighbors.length > 0) {
-        layerConns.push([layer, liveNeighbors]);
-      }
-    }
-    nodeArray.push([docId, maxLayer, layerConns]);
-  }
-  const entryOrd = entryPointOf(state);
-  const entryPoint = entryOrd === -1 ? null : store.docIdForOrdinal(entryOrd) ?? null;
-  return {
-    entryPoint,
-    maxLayer: topLayerOf(state),
-    m: state.M,
-    efConstruction: state.efCons,
-    metric: state.buildMetric,
-    nodes: nodeArray
-  };
-}
 function resolveNodes(state, store, data) {
   const nodes = [];
   let maxOrd = -1;
@@ -4186,13 +4186,12 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
     compactTombstones: () => exclusively(() => compactTombstones(state)),
     rebuild: () => exclusively(() => rebuild(state)),
     exclusively,
-    serialize: () => serializeGraph(state, store),
     deserialize: (data) => exclusively(() => deserializeGraph(state, store, data)),
-    serializeNumbered: (numberOfOrdinal, ordinalOfNumber) => serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber),
+    serializeNumbered: (numberOfOrdinal, ordinalOfNumber, partSize) => serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize),
     deserializeNumbered: (graph, ordinalOfNumber) => exclusively(() => deserializeNumberedGraph(state, graph, ordinalOfNumber)),
     exportSnapshot: () => exportSnapshot(state),
     restoreSnapshot: (snapshot) => exclusively(() => restoreSnapshot(state, snapshot))
   };
 }
 
-export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore };
+export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };