@delali/narsil 0.3.1-next.71 → 0.3.1-next.73

package/dist/{chunk-YJDQ7NLA.mjs → chunk-WVDFG45M.mjs}

@@ -1,7 +1,7 @@
 import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
-import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-YNVIUGF5.mjs';
-import { compareCodePoints } from './chunk-OI42GWLR.mjs';
-import { NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
+import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-PU43WDY4.mjs';
+import { compareCodePoints } from './chunk-LJYTW7GT.mjs';
+import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
 import { readSync, closeSync, openSync } from 'fs';
 
 // src/vector/constants.ts
@@ -3651,6 +3651,13 @@ function createVectorStore(options) {
         yield [docId, entryAt(open, ordinal)];
       }
     },
+    *docIds() {
+      if (open === null) return;
+      for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
+        const docId = ordToDoc[ordinal];
+        if (docId !== void 0) yield docId;
+      }
+    },
     clear() {
       forgetDocuments();
       if (open === null) return;
@@ -3789,30 +3796,24 @@ function savedTopLayer(state, ordinal) {
   if (ordinal === NO_ORDINAL || isTombstoned(state, ordinal)) return NO_LAYER;
   return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
 }
-function largestNeighbourValueCount(state, topLayers) {
+function neighbourValueCount(state, topLayers, start, end) {
   let values = 0;
-  for (let number = 0; number < topLayers.length; number++) {
+  for (let number = start; number < end; number++) {
     const top = topLayers[number];
     if (top < 0) continue;
     values += state.adjacency.level0Stride + top * state.adjacency.upperStride;
   }
   return values;
 }
-function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
+function serializePart(state, numbers, start, end) {
+  const { numberOfOrdinal, ordinalOfNumber, topLayers } = numbers;
   const adjacency = state.adjacency;
-  const slots = adjacencySlots(adjacency);
-  const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
-  const levels = new Uint8Array(ordinalOfNumber.length);
-  for (let number = 0; number < ordinalOfNumber.length; number++) {
-    const ordinal = ordinalOfNumber[number];
-    if (ordinal >= slots) continue;
-    topLayers[number] = savedTopLayer(state, ordinal);
-    levels[number] = topLayers[number] + 1;
-  }
-  const values = new Uint32Array(largestNeighbourValueCount(state, topLayers));
+  const levels = new Uint8Array(end - start);
+  const values = new Uint32Array(neighbourValueCount(state, topLayers, start, end));
   let cursor = 0;
-  for (let number = 0; number < ordinalOfNumber.length; number++) {
+  for (let number = start; number < end; number++) {
     const top = topLayers[number];
+    levels[number - start] = top + 1;
     for (let layer = 0; layer <= top; layer++) {
       const base = layerBase(adjacency, ordinalOfNumber[number], layer);
       const array = layerArray(adjacency, layer);
@@ -3830,18 +3831,33 @@ function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
       values[countAt] = cursor - countAt - 1;
     }
   }
+  return { levels, neighbours: littleEndianBytesOf(values.subarray(0, cursor)) };
+}
+function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize) {
+  const slots = adjacencySlots(state.adjacency);
+  const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
+  for (let number = 0; number < ordinalOfNumber.length; number++) {
+    const ordinal = ordinalOfNumber[number];
+    if (ordinal >= slots) continue;
+    topLayers[number] = savedTopLayer(state, ordinal);
+  }
   const entryOrdinal = entryPointOf(state);
   const entryNumber = entryOrdinal >= 0 && entryOrdinal < numberOfOrdinal.length ? numberOfOrdinal[entryOrdinal] : NO_NUMBER;
   const entryHoldsNode = entryNumber !== NO_NUMBER && topLayers[entryNumber] >= 0;
-  return {
+  const header = {
     entryPoint: entryHoldsNode ? entryNumber : null,
     maxLayer: Math.max(0, Math.min(topLayerOf(state), MAX_LAYER_CAP)),
     m: state.M,
     efConstruction: state.efCons,
-    metric: state.buildMetric,
-    levels,
-    neighbours: littleEndianBytesOf(values.subarray(0, cursor))
+    metric: state.buildMetric
   };
+  const numbers = { numberOfOrdinal, ordinalOfNumber, topLayers };
+  const parts = [];
+  for (let start = 0; start < ordinalOfNumber.length; start += partSize) {
+    const end = Math.min(start + partSize, ordinalOfNumber.length);
+    parts.push({ ...header, ...serializePart(state, numbers, start, end) });
+  }
+  return parts;
 }
 function endsInsideAList() {
   throw new NarsilError(
@@ -3862,95 +3878,78 @@ function requireWholeNeighbourLists(graph) {
     }
   }
 }
-function topLayerIn(graph, number) {
-  return number < graph.levels.length ? graph.levels[number] - 1 : -1;
+function topLayerInParts(graph, number) {
+  const part = graph.parts[Math.floor(number / graph.partSize)];
+  if (part === void 0) return -1;
+  const at = number % graph.partSize;
+  return at < part.levels.length ? part.levels[at] - 1 : -1;
+}
+function highestNodeOrdinal(graph, ordinalAt) {
+  let highest = -1;
+  for (let p = 0; p < graph.parts.length; p++) {
+    const levels = graph.parts[p].levels;
+    for (let at = 0; at < levels.length; at++) {
+      const ordinal = ordinalAt(p * graph.partSize + at);
+      if (levels[at] > 0 && ordinal > highest) highest = ordinal;
+    }
+  }
+  return highest;
 }
 function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
-  requireWholeNeighbourLists(graph);
+  for (const part of graph.parts) requireWholeNeighbourLists(part);
   resetGraph(state);
-  const values = valuesOf(graph.neighbours);
   const ordinalAt = (number) => number < ordinalOfNumber.length ? ordinalOfNumber[number] : NO_ORDINAL;
-  let highestOrdinal = -1;
-  for (let number = 0; number < graph.levels.length; number++) {
-    if (graph.levels[number] > 0 && ordinalAt(number) > highestOrdinal) highestOrdinal = ordinalAt(number);
-  }
+  const highestOrdinal = highestNodeOrdinal(graph, ordinalAt);
   if (highestOrdinal >= 0) ensureCapacity(state, highestOrdinal + 1);
   const kept = [];
-  let cursor = 0;
   let highestLayer = -1;
   let nodeOnHighestLayer = NO_ORDINAL;
-  for (let number = 0; number < graph.levels.length; number++) {
-    const top = graph.levels[number] - 1;
-    if (top < 0) continue;
-    const ordinal = ordinalAt(number);
-    const keptTop = Math.min(top, MAX_LAYER_CAP);
-    if (ordinal !== NO_ORDINAL) {
-      createNode(state.adjacency, ordinal, keptTop);
-      Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
-      if (keptTop > highestLayer) {
-        highestLayer = keptTop;
-        nodeOnHighestLayer = ordinal;
-      }
-    }
-    for (let layer = 0; layer <= top; layer++) {
-      const count = values[cursor];
-      kept.length = 0;
-      const limit = maxConns(state, layer);
-      for (let i = 1; i <= count && kept.length < limit; i++) {
-        const neighbour = values[cursor + i];
-        const neighbourOrdinal = ordinalAt(neighbour);
-        if (neighbourOrdinal === NO_ORDINAL || neighbour === number || topLayerIn(graph, neighbour) < layer) continue;
-        if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
+  for (let p = 0; p < graph.parts.length; p++) {
+    const part = graph.parts[p];
+    const values = valuesOf(part.neighbours);
+    let cursor = 0;
+    for (let at = 0; at < part.levels.length; at++) {
+      const number = p * graph.partSize + at;
+      const top = part.levels[at] - 1;
+      if (top < 0) continue;
+      const ordinal = ordinalAt(number);
+      const keptTop = Math.min(top, MAX_LAYER_CAP);
+      if (ordinal !== NO_ORDINAL) {
+        createNode(state.adjacency, ordinal, keptTop);
+        Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
+        if (keptTop > highestLayer) {
+          highestLayer = keptTop;
+          nodeOnHighestLayer = ordinal;
+        }
       }
-      cursor += count + 1;
-      if (ordinal !== NO_ORDINAL && layer <= keptTop) {
-        replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
+      for (let layer = 0; layer <= top; layer++) {
+        const count = values[cursor];
+        kept.length = 0;
+        const limit = maxConns(state, layer);
+        for (let i = 1; i <= count && kept.length < limit; i++) {
+          const neighbour = values[cursor + i];
+          const neighbourOrdinal = ordinalAt(neighbour);
+          if (neighbourOrdinal === NO_ORDINAL || neighbour === number) continue;
+          if (topLayerInParts(graph, neighbour) < layer) continue;
+          if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
+        }
+        cursor += count + 1;
+        if (ordinal !== NO_ORDINAL && layer <= keptTop) {
+          replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
+        }
       }
     }
   }
-  const entryOrdinal = graph.entryPoint === null ? NO_ORDINAL : ordinalAt(graph.entryPoint);
-  const entryTop = graph.entryPoint === null ? -1 : Math.min(topLayerIn(graph, graph.entryPoint), MAX_LAYER_CAP);
+  const first = graph.parts[0];
+  const entryPoint = first === void 0 ? null : first.entryPoint;
+  const entryOrdinal = entryPoint === null ? NO_ORDINAL : ordinalAt(entryPoint);
+  const entryTop = entryPoint === null ? -1 : Math.min(topLayerInParts(graph, entryPoint), MAX_LAYER_CAP);
   const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
   Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
   Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
 }
 
 // src/vector/hnsw/persistence.ts
-function serializeGraph(state, store) {
-  const nodeArray = [];
-  const slots = adjacencySlots(state.adjacency);
-  for (let ord = 0; ord < slots; ord++) {
-    const maxLayer = nodeMaxLayer(state, ord);
-    if (maxLayer === -1) continue;
-    if (isTombstoned(state, ord)) continue;
-    const docId = store.docIdForOrdinal(ord);
-    if (docId === void 0) continue;
-    const layerConns = [];
-    for (let layer = 0; layer <= maxLayer; layer++) {
-      const liveNeighbors = [];
-      for (const neighborOrd of collectNeighbors(state.adjacency, ord, layer)) {
-        if (isTombstoned(state, neighborOrd)) continue;
-        const neighborDoc = store.docIdForOrdinal(neighborOrd);
-        if (neighborDoc === void 0) continue;
-        liveNeighbors.push(neighborDoc);
-      }
-      if (liveNeighbors.length > 0) {
-        layerConns.push([layer, liveNeighbors]);
-      }
-    }
-    nodeArray.push([docId, maxLayer, layerConns]);
-  }
-  const entryOrd = entryPointOf(state);
-  const entryPoint = entryOrd === -1 ? null : store.docIdForOrdinal(entryOrd) ?? null;
-  return {
-    entryPoint,
-    maxLayer: topLayerOf(state),
-    m: state.M,
-    efConstruction: state.efCons,
-    metric: state.buildMetric,
-    nodes: nodeArray
-  };
-}
 function resolveNodes(state, store, data) {
   const nodes = [];
   let maxOrd = -1;
@@ -4186,13 +4185,12 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
     compactTombstones: () => exclusively(() => compactTombstones(state)),
     rebuild: () => exclusively(() => rebuild(state)),
     exclusively,
-    serialize: () => serializeGraph(state, store),
     deserialize: (data) => exclusively(() => deserializeGraph(state, store, data)),
-    serializeNumbered: (numberOfOrdinal, ordinalOfNumber) => serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber),
+    serializeNumbered: (numberOfOrdinal, ordinalOfNumber, partSize) => serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize),
     deserializeNumbered: (graph, ordinalOfNumber) => exclusively(() => deserializeNumberedGraph(state, graph, ordinalOfNumber)),
     exportSnapshot: () => exportSnapshot(state),
     restoreSnapshot: (snapshot) => exclusively(() => restoreSnapshot(state, snapshot))
   };
 }
 
-export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore };
+export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };