@delali/narsil 0.2.4-next.51 → 0.2.4-next.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +27 -25
  2. package/dist/{capabilities-BWm4IITU.d.ts → capabilities-CuTSXvbN.d.ts} +2 -2
  3. package/dist/{chunk-BEIK3EOZ.mjs → chunk-2FO7KZ6K.mjs} +5468 -3837
  4. package/dist/{chunk-XE6XR2OE.mjs → chunk-3GWP2TO6.mjs} +1 -1
  5. package/dist/{chunk-KMMJMDUA.mjs → chunk-3L4U6EJQ.mjs} +15 -10
  6. package/dist/{chunk-MHE7YWAU.mjs → chunk-72AXTIBW.mjs} +2 -2
  7. package/dist/{chunk-CV5H7XL7.mjs → chunk-EY2ULUMT.mjs} +30 -7
  8. package/dist/chunk-IFBKLBGB.mjs +7 -0
  9. package/dist/{chunk-WYJXSLER.mjs → chunk-POWOTUKJ.mjs} +7 -1
  10. package/dist/chunk-T62YOSGS.mjs +4280 -0
  11. package/dist/chunk-U36IDJN7.mjs +366 -0
  12. package/dist/{chunk-ND34I3SV.mjs → chunk-ULJ36FK4.mjs} +3502 -2383
  13. package/dist/{chunk-RP23PWMV.mjs → chunk-WFKIGEBY.mjs} +3 -3
  14. package/dist/{chunk-CMVG2UEF.mjs → chunk-WM7IS45M.mjs} +55 -517
  15. package/dist/chunk-Y3L5MDRV.mjs +96 -0
  16. package/dist/{chunk-ZHUQKZBE.mjs → chunk-YPYULM5E.mjs} +20 -4
  17. package/dist/chunk-ZJ4CSAB2.mjs +13 -0
  18. package/dist/chunk-ZT5BRDFW.mjs +513 -0
  19. package/dist/{client-B_X6_AbT.d.ts → client-BMJWVxOP.d.ts} +3 -3
  20. package/dist/client.d.ts +8 -8
  21. package/dist/{config-OON7IJ_B.d.ts → config-BcYZkdMw.d.ts} +2 -2
  22. package/dist/{config-Czh2YNDi.d.ts → config-qMLpi5T4.d.ts} +2 -2
  23. package/dist/direct-executor-UJC36S2D.mjs +12 -0
  24. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  25. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  26. package/dist/distribution.d.ts +8 -8
  27. package/dist/distribution.mjs +30 -15
  28. package/dist/{engine-9PtFY50q.d.ts → engine-zhms1qId.d.ts} +2 -2
  29. package/dist/index.browser.mjs +12845 -7766
  30. package/dist/index.d.ts +9 -21
  31. package/dist/index.mjs +13 -12
  32. package/dist/{memory-Bzd5xcEW.d.ts → memory-OmgkX_Z2.d.ts} +11 -1
  33. package/dist/persistence/durability/checkpoint-worker.d.ts +138 -3
  34. package/dist/persistence/durability/checkpoint-worker.mjs +31 -17
  35. package/dist/react.d.ts +6 -6
  36. package/dist/{schema-BqRx9t4E.d.ts → schema-OSWaaZzx.d.ts} +30 -6
  37. package/dist/serialization/crc32-worker.mjs +2 -1
  38. package/dist/server.d.ts +10 -10
  39. package/dist/server.mjs +75 -34
  40. package/dist/simd-DW5AtJ6B.d.ts +41 -0
  41. package/dist/{tasks-DT7L1p6w.d.ts → tasks-DiF1c9v2.d.ts} +1 -1
  42. package/dist/{thread-PZLEO6JL.mjs → thread-TLCX43WV.mjs} +129 -41
  43. package/dist/{types-DXPxaXPO.d.ts → types-LEzMyjVX.d.ts} +1 -1
  44. package/dist/vector/search-worker.d.ts +292 -128
  45. package/dist/vector/search-worker.mjs +70 -54
  46. package/dist/workers/entry.mjs +10 -7
  47. package/package.json +52 -31
  48. package/dist/brute-force-DbHYCTc3.d.ts +0 -8
  49. package/dist/chunk-CBLS46IE.mjs +0 -10
  50. package/dist/chunk-PIWKZPP7.mjs +0 -64
  51. package/dist/chunk-PMBHH44R.mjs +0 -492
  52. package/dist/chunk-X6YZO4MQ.mjs +0 -1572
  53. package/dist/direct-executor-OREXU72C.mjs +0 -12
  54. package/dist/vector/hnsw-build-worker.d.ts +0 -105
  55. package/dist/vector/hnsw-build-worker.mjs +0 -123
@@ -1,1572 +0,0 @@
1
- import { INITIAL_LIST_CAPACITY, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR, INITIAL_UPPER_SLOTS } from './chunk-RP23PWMV.mjs';
2
- import { compareCodePoints } from './chunk-2GWVQWGA.mjs';
3
-
4
- // src/vector/constants.ts
5
- var WASM_PAGE_BYTES = 65536;
6
- var MAX_WASM_PAGES = 65536;
7
- var VECTOR_STORE_INITIAL_CAPACITY = 16;
8
- var MAX_WORKER_DIMENSION = 8192;
9
- var MAX_WORKER_VECTORS = 1e6;
10
- var MAX_WORKER_COPIES = 64;
11
- var HNSW_BUILD_TIMEOUT_MS = 12e4;
12
- var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
13
- var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
14
- var QUANTIZATION_PADDING_FACTOR = 0.01;
15
-
16
- // src/vector/simd-wasm-binary.ts
17
- var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABFQNgA39/fwF9YAJ/fwF9YAN/f38BfwMHBgABAAACAgUDAQABB2sHBm1lbW9yeQIAC2RvdF9wcm9kdWN0AAAJbWFnbml0dWRlAAEac3F1YXJlZF9ldWNsaWRlYW5fZGlzdGFuY2UAAhJldWNsaWRlYW5fZGlzdGFuY2UAAwZkb3RfdTgABAlzcWRpc3RfdTgABQqvBwauAQQBfwF7AX0BfyACQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhBEEAIQMCQANAIAMgBk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAE/R8AIAT9HwGSIAT9HwIgBP0fA5KSIQUCQANAIAMgAk8NASAFIAAgA0ECdGoqAgAgASADQQJ0aioCAJSSIQUgA0EBaiEDDAALCyAFC6kBBAF/AnsBfQF/IAFBfHEhBv0MAAAAAAAAAAAAAAAAAAAAACEDQQAhAgJAA0AgAiAGTw0BIAAgAkECdGr9AAQAIQQgAyAEIAT95gH95AEhAyACQQRqIQIMAAsLIAP9HwAgA/0fAZIgA/0fAiAD/R8DkpIhBQJAA0AgAiABTw0BIAUgACACQQJ0aioCACAAIAJBAnRqKgIAlJIhBSACQQFqIQIMAAsLIAWRC8ABBQF/AnsBfQF/AX0gAkF8cSEH/QwAAAAAAAAAAAAAAAAAAAAAIQRBACEDAkADQCADIAdPDQEgACADQQJ0av0ABAAgASADQQJ0av0ABAD95QEhBSAEIAUgBf3mAf3kASEEIANBBGohAwwACwsgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEGAkADQCADIAJPDQEgACADQQJ0aioCACABIANBAnRqKgIAkyEIIAYgCCAIlJIhBiADQQFqIQMMAAsLIAYLCwAgACABIAIQApELtAEDAX8DewJ/IAJBcHEhCEEA/REhBEEAIQMCQANAIAMgCE8NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBCAF/YkBIAb9iQH9ugH9rgEhBCAEIAX9igEgBv2KAf26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEHAkADQCADIAJPDQEgByAAIANqLQAAIAEgA2otAABsaiEHIANBAWohAwwACwsgBwvNAQMBfwV7A38gAkFwcSELQQD9ESEEQQAhAwJAA0AgAyALTw0BIAAgA2r9AAQAIQUgASADav0ABAAhBiAF/YkBIAb9iQH9kQEhByAF/YoBIAb9igH9kQEhCCAEIAcgB/26Af2uASEEIAQgCCAI/boB/a4BIQQgA0EQaiEDDAALCyAE/RsAIAT9GwFqIAT9GwIgBP0bA2pqIQkCQANAIAMgAk8NASAAIANqLQAAIAEgA2otAABrIQogCSAKIApsaiEJIANBAWohAwwACwsgCQs=";
18
- var SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64 = "AGFzbQEAAAABFQNgA39/fwF9YAJ/fwF9YAN/f38BfwISAQNlbnYGbWVtb3J5AgMBgIAEAwcGAAEAAAICB2IGC2RvdF9wcm9kdWN0AAAJbWFnbml0dWRlAAEac3F1YXJlZF9ldWNsaWRlYW5fZGlzdGFuY2UAAhJldWNsaWRlYW5fZGlzdGFuY2UAAwZkb3RfdTgABAlzcWRpc3RfdTgABQqvBwauAQQBfwF7AX0BfyACQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhBEEAIQMCQANAIAMgBk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAE/R8AIAT9HwGSIAT9HwIgBP0fA5KSIQUCQANAIAMgAk8NASAFIAAgA0ECdGoqAgAgASADQQJ0aioCAJSSIQUgA0EBaiEDDAALCyAFC6kBBAF/AnsBfQF/IAFBfHEhBv0MAAAAAAAAAAAAAAAAAAAAACEDQQAhAgJAA0AgAiAGTw0BIAAgAkECdGr9AAQAIQQgAyAEIAT95gH95AEhAyACQQRqIQIMAAsLIAP9HwAgA/0fAZIgA/0fAiAD/R8DkpIhBQJAA0AgAiABTw0BIAUgACACQQJ0aioCACAAIAJBAnRqKgIAlJIhBSACQQFqIQIMAAsLIAWRC8ABBQF/AnsBfQF/AX0gAkF8cSEH/QwAAAAAAAAAAAAAAAAAAAAAIQRBACEDAkADQCADIAdPDQEgACADQQJ0av0ABAAgASADQQJ0av0ABAD95QEhBSAEIAUgBf3mAf3kASEEIANBBGohAwwACwsgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEGAkADQCADIAJPDQEgACADQQJ0aioCACABIANBAnRqKgIAkyEIIAYgCCAIlJIhBiADQQFqIQMMAAsLIAYLCwAgACABIAIQApELtAEDAX8DewJ/IAJBcHEhCEEA/REhBEEAIQMCQANAIAMgCE8NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBCAF/YkBIAb9iQH9ugH9rgEhBCAEIAX9igEgBv2KAf26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEHAkADQCADIAJPDQEgByAAIANqLQAAIAEgA2otAABsaiEHIANBAWohAwwACwsgBwvNAQMBfwV7A38gAkFwcSELQQD9ESEEQQAhAwJAA0AgAyALTw0BIAAgA2r9AAQAIQUgASADav0ABAAhBiAF/YkBIAb9iQH9kQEhByAF/YoBIAb9igH9kQEhCCAEIAcgB/26Af2uASEEIAQgCCAI/boB/a4BIQQgA0EQaiEDDAALCyAE/RsAIAT9GwFqIAT9GwIgBP0bA2pqIQkCQANAIAMgAk8NASAAIANqLQAAIAEgA2otAABrIQogCSAKIApsaiEJIANBAWohAwwACwsgCQs=";
19
-
20
- // src/vector/simd.ts
21
- var wasmExports = null;
22
- var f32View = null;
23
- function decodeBase64(base64) {
24
- if (typeof Buffer !== "undefined") {
25
- return new Uint8Array(Buffer.from(base64, "base64"));
26
- }
27
- const binary = atob(base64);
28
- const bytes = new Uint8Array(binary.length);
29
- for (let i = 0; i < binary.length; i++) {
30
- bytes[i] = binary.charCodeAt(i);
31
- }
32
- return bytes;
33
- }
34
- try {
35
- const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
36
- const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
37
- if (typeof WebAssembly !== "undefined" && typeof WebAssembly.validate === "function" && WebAssembly.validate(wasmBuffer)) {
38
- const module = new WebAssembly.Module(wasmBuffer);
39
- const instance = new WebAssembly.Instance(module);
40
- wasmExports = instance.exports;
41
- f32View = new Float32Array(wasmExports.memory.buffer);
42
- }
43
- } catch {
44
- wasmExports = null;
45
- f32View = null;
46
- }
47
- var OFFSET_B_BYTES = 16384;
48
- function ensureMemory(dim) {
49
- if (!wasmExports || !f32View) return;
50
- const needed = OFFSET_B_BYTES + dim * 4;
51
- const currentSize = wasmExports.memory.buffer.byteLength;
52
- if (needed > currentSize) {
53
- const pagesNeeded = Math.ceil(needed / 65536);
54
- const currentPages = currentSize / 65536;
55
- wasmExports.memory.grow(pagesNeeded - currentPages);
56
- f32View = new Float32Array(wasmExports.memory.buffer);
57
- }
58
- }
59
- function getF32View() {
60
- if (!f32View) {
61
- throw new Error("SIMD memory is not initialized");
62
- }
63
- return f32View;
64
- }
65
- function copyOneVector(a) {
66
- getF32View().set(a, 0);
67
- }
68
- function copyTwoVectors(a, b) {
69
- const view = getF32View();
70
- view.set(a, 0);
71
- view.set(b, OFFSET_B_BYTES / 4);
72
- }
73
- function simdDotProduct(a, b) {
74
- if (!wasmExports || !f32View) return null;
75
- ensureMemory(a.length);
76
- copyTwoVectors(a, b);
77
- return wasmExports.dot_product(0, OFFSET_B_BYTES, a.length);
78
- }
79
- function simdMagnitude(a) {
80
- if (!wasmExports || !f32View) return null;
81
- ensureMemory(a.length);
82
- copyOneVector(a);
83
- return wasmExports.magnitude(0, a.length);
84
- }
85
- function simdEuclideanDistance(a, b) {
86
- if (!wasmExports || !f32View) return null;
87
- ensureMemory(a.length);
88
- copyTwoVectors(a, b);
89
- return wasmExports.euclidean_distance(0, OFFSET_B_BYTES, a.length);
90
- }
91
- function isSimdAvailable() {
92
- return wasmExports !== null;
93
- }
94
- var arenaModule;
95
- function getArenaModule() {
96
- if (arenaModule !== void 0) return arenaModule;
97
- try {
98
- const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
99
- const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
100
- if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
101
- arenaModule = null;
102
- return null;
103
- }
104
- arenaModule = new WebAssembly.Module(wasmBuffer);
105
- } catch {
106
- arenaModule = null;
107
- }
108
- return arenaModule;
109
- }
110
- function createArenaSimd() {
111
- try {
112
- const module = getArenaModule();
113
- if (module === null) return null;
114
- const instance = new WebAssembly.Instance(module);
115
- const exports$1 = instance.exports;
116
- if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function" || !(exports$1.memory instanceof WebAssembly.Memory)) {
117
- return null;
118
- }
119
- return exports$1;
120
- } catch {
121
- return null;
122
- }
123
- }
124
- var sharedMemoryModule;
125
- function getSharedMemoryModule() {
126
- if (sharedMemoryModule !== void 0) return sharedMemoryModule;
127
- try {
128
- const bytes = decodeBase64(SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64);
129
- const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
130
- if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
131
- sharedMemoryModule = null;
132
- return null;
133
- }
134
- sharedMemoryModule = new WebAssembly.Module(wasmBuffer);
135
- } catch {
136
- sharedMemoryModule = null;
137
- }
138
- return sharedMemoryModule;
139
- }
140
- function createSharedArenaSimd(memory) {
141
- try {
142
- const module = getSharedMemoryModule();
143
- if (module === null) return null;
144
- const instance = new WebAssembly.Instance(module, { env: { memory } });
145
- const exports$1 = instance.exports;
146
- if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function") {
147
- return null;
148
- }
149
- return {
150
- memory,
151
- dot_product: exports$1.dot_product,
152
- magnitude: exports$1.magnitude,
153
- squared_euclidean_distance: exports$1.squared_euclidean_distance,
154
- dot_u8: exports$1.dot_u8,
155
- sqdist_u8: exports$1.sqdist_u8
156
- };
157
- } catch {
158
- return null;
159
- }
160
- }
161
- function arenaFloat32Distance(simd, byteA, byteB, dimension, metric, magnitudeA, magnitudeB) {
162
- if (metric === "euclidean") {
163
- return Math.sqrt(simd.squared_euclidean_distance(byteA, byteB, dimension));
164
- }
165
- const dot = simd.dot_product(byteA, byteB, dimension);
166
- if (metric === "dotProduct") return -dot;
167
- if (magnitudeA === 0 || magnitudeB === 0) return 1;
168
- return 1 - dot / (magnitudeA * magnitudeB);
169
- }
170
-
171
- // src/vector/ordinal-filter.ts
172
- function createOrdinalFilter(slots) {
173
- return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
174
- }
175
- function addToOrdinalFilter(filter, ordinal) {
176
- const index = ordinal >>> 3;
177
- if (index >= filter.bits.length) return;
178
- const bit = 1 << (ordinal & 7);
179
- if ((filter.bits[index] & bit) !== 0) return;
180
- filter.bits[index] |= bit;
181
- filter.count += 1;
182
- }
183
- function ordinalFilterHas(filter, ordinal) {
184
- const index = ordinal >>> 3;
185
- if (index >= filter.bits.length) return false;
186
- return (filter.bits[index] & 1 << (ordinal & 7)) !== 0;
187
- }
188
- function removeFromOrdinalFilter(filter, ordinal) {
189
- const index = ordinal >>> 3;
190
- if (index >= filter.bits.length) return;
191
- const bit = 1 << (ordinal & 7);
192
- if ((filter.bits[index] & bit) === 0) return;
193
- filter.bits[index] &= ~bit;
194
- filter.count -= 1;
195
- }
196
- function* ordinalFilterValues(filter) {
197
- const bits = filter.bits;
198
- for (let index = 0; index < bits.length; index++) {
199
- const byte = bits[index];
200
- if (byte === 0) continue;
201
- for (let offset = 0; offset < 8; offset++) {
202
- if ((byte & 1 << offset) !== 0) yield index << 3 | offset;
203
- }
204
- }
205
- }
206
-
207
- // src/vector/similarity.ts
208
- function magnitude(v) {
209
- const result = simdMagnitude(v);
210
- if (result !== null) return result;
211
- let sum = 0;
212
- for (let i = 0; i < v.length; i++) {
213
- sum += v[i] * v[i];
214
- }
215
- return Math.sqrt(sum);
216
- }
217
- function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
218
- if (magA === 0 || magB === 0) return 0;
219
- return dotProduct(a, b) / (magA * magB);
220
- }
221
- function dotProduct(a, b) {
222
- const result = simdDotProduct(a, b);
223
- if (result !== null) return result;
224
- let sum = 0;
225
- for (let i = 0; i < a.length; i++) {
226
- sum += a[i] * b[i];
227
- }
228
- return sum;
229
- }
230
- function euclideanDistance(a, b) {
231
- const result = simdEuclideanDistance(a, b);
232
- if (result !== null) return result;
233
- let sum = 0;
234
- for (let i = 0; i < a.length; i++) {
235
- const diff = a[i] - b[i];
236
- sum += diff * diff;
237
- }
238
- return Math.sqrt(sum);
239
- }
240
-
241
- // src/vector/vector-store/index.ts
242
- var UNKNOWN_PARTITION = -1;
243
- var NO_PARTITION = -2;
244
- function createVectorStore() {
245
- const docToOrd = /* @__PURE__ */ new Map();
246
- const recycledOrds = /* @__PURE__ */ new Map();
247
- const ordToDoc = [];
248
- let partitionOf = new Int32Array(0);
249
- let unknownPartitions = 0;
250
- let simd = createArenaSimd();
251
- let dimension = 0;
252
- let capacity = 0;
253
- let arena = new Float32Array(0);
254
- let mags = new Float64Array(0);
255
- let liveCount = 0;
256
- let scratchByteLength = 0;
257
- let scratchFloatLength = 0;
258
- function initStorage(dim) {
259
- dimension = dim;
260
- if (simd) {
261
- scratchByteLength = Math.max(16, Math.ceil(dim * 4 / 16) * 16);
262
- scratchFloatLength = scratchByteLength / 4;
263
- arena = new Float32Array(simd.memory.buffer);
264
- }
265
- }
266
- function ensureCapacity2(needed) {
267
- if (needed <= capacity) return;
268
- let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
269
- while (newCap < needed) newCap *= 2;
270
- const nextMags = new Float64Array(newCap);
271
- nextMags.set(mags);
272
- mags = nextMags;
273
- if (simd) {
274
- const requiredBytes = scratchByteLength + newCap * dimension * 4;
275
- const have = simd.memory.buffer.byteLength;
276
- if (requiredBytes > have) {
277
- try {
278
- simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
279
- } catch {
280
- const migrated = new Float32Array(newCap * dimension);
281
- migrated.set(arena.subarray(scratchFloatLength, scratchFloatLength + capacity * dimension));
282
- arena = migrated;
283
- simd = null;
284
- scratchByteLength = 0;
285
- scratchFloatLength = 0;
286
- capacity = newCap;
287
- return;
288
- }
289
- }
290
- arena = new Float32Array(simd.memory.buffer);
291
- } else {
292
- const nextArena = new Float32Array(newCap * dimension);
293
- nextArena.set(arena.subarray(0, capacity * dimension));
294
- arena = nextArena;
295
- }
296
- capacity = newCap;
297
- }
298
- function writeVector(ord, vector) {
299
- arena.set(vector, scratchFloatLength + ord * dimension);
300
- mags[ord] = magnitude(vector);
301
- }
302
- function entryAt(ord) {
303
- const base = scratchFloatLength + ord * dimension;
304
- return { vector: arena.subarray(base, base + dimension), magnitude: mags[ord] };
305
- }
306
- function ensurePartitionSlots(ord) {
307
- if (ord < partitionOf.length) return;
308
- let length = partitionOf.length === 0 ? VECTOR_STORE_INITIAL_CAPACITY : partitionOf.length;
309
- while (length <= ord) length *= 2;
310
- const grown = new Int32Array(length).fill(UNKNOWN_PARTITION);
311
- grown.set(partitionOf);
312
- partitionOf = grown;
313
- }
314
- function recordPartition(ord, partitionId, wasLive) {
315
- ensurePartitionSlots(ord);
316
- const previous = partitionOf[ord];
317
- const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : void 0;
318
- const next = given ?? (wasLive ? previous : UNKNOWN_PARTITION);
319
- partitionOf[ord] = next;
320
- if (!wasLive && next === UNKNOWN_PARTITION) {
321
- unknownPartitions += 1;
322
- return;
323
- }
324
- if (wasLive && previous === UNKNOWN_PARTITION && next !== UNKNOWN_PARTITION) {
325
- unknownPartitions -= 1;
326
- }
327
- }
328
- return {
329
- get size() {
330
- return liveCount;
331
- },
332
- get slots() {
333
- return ordToDoc.length;
334
- },
335
- insert(docId, vector, partitionId) {
336
- if (dimension === 0) initStorage(vector.length);
337
- const existing = docToOrd.get(docId);
338
- if (existing !== void 0) {
339
- writeVector(existing, vector);
340
- recordPartition(existing, partitionId, true);
341
- return;
342
- }
343
- const recycled = recycledOrds.get(docId);
344
- if (recycled !== void 0) {
345
- recycledOrds.delete(docId);
346
- docToOrd.set(docId, recycled);
347
- ordToDoc[recycled] = docId;
348
- ensureCapacity2(recycled + 1);
349
- writeVector(recycled, vector);
350
- recordPartition(recycled, partitionId, false);
351
- liveCount++;
352
- return;
353
- }
354
- const ord = ordToDoc.length;
355
- docToOrd.set(docId, ord);
356
- ordToDoc.push(docId);
357
- ensureCapacity2(ord + 1);
358
- writeVector(ord, vector);
359
- recordPartition(ord, partitionId, false);
360
- liveCount++;
361
- },
362
- setPartition(docId, partitionId) {
363
- const ord = docToOrd.get(docId);
364
- if (ord === void 0) return;
365
- recordPartition(ord, partitionId, true);
366
- },
367
- forgetPartition(docId) {
368
- const ord = docToOrd.get(docId);
369
- if (ord === void 0 || ord >= partitionOf.length) return;
370
- if (partitionOf[ord] === UNKNOWN_PARTITION) {
371
- unknownPartitions -= 1;
372
- }
373
- partitionOf[ord] = NO_PARTITION;
374
- },
375
- partitionOfOrdinal(ordinal) {
376
- if (ordinal < 0 || ordinal >= partitionOf.length) return void 0;
377
- const partitionId = partitionOf[ordinal];
378
- return partitionId < 0 ? void 0 : partitionId;
379
- },
380
- get partitionsKnown() {
381
- return unknownPartitions === 0;
382
- },
383
- partitionFilter(partitionIds) {
384
- const filter = createOrdinalFilter(ordToDoc.length);
385
- let highest = -1;
386
- for (const partitionId of partitionIds) {
387
- if (partitionId > highest) highest = partitionId;
388
- }
389
- if (highest < 0) return filter;
390
- const wanted = new Uint8Array(highest + 1);
391
- for (const partitionId of partitionIds) {
392
- if (partitionId >= 0) wanted[partitionId] = 1;
393
- }
394
- const upperBound = Math.min(ordToDoc.length, partitionOf.length);
395
- for (let ord = 0; ord < upperBound; ord++) {
396
- if (ordToDoc[ord] === void 0) continue;
397
- const partitionId = partitionOf[ord];
398
- if (partitionId < 0 || partitionId > highest || wanted[partitionId] === 0) continue;
399
- addToOrdinalFilter(filter, ord);
400
- }
401
- return filter;
402
- },
403
- remove(docId) {
404
- const ord = docToOrd.get(docId);
405
- if (ord === void 0) return;
406
- docToOrd.delete(docId);
407
- ordToDoc[ord] = void 0;
408
- recycledOrds.set(docId, ord);
409
- if (ord < partitionOf.length && partitionOf[ord] === UNKNOWN_PARTITION) {
410
- unknownPartitions -= 1;
411
- }
412
- liveCount--;
413
- },
414
- get(docId) {
415
- const ord = docToOrd.get(docId);
416
- return ord === void 0 ? void 0 : entryAt(ord);
417
- },
418
- has(docId) {
419
- return docToOrd.has(docId);
420
- },
421
- *entries() {
422
- for (let ord = 0; ord < ordToDoc.length; ord++) {
423
- const docId = ordToDoc[ord];
424
- if (docId === void 0) continue;
425
- yield [docId, entryAt(ord)];
426
- }
427
- },
428
- clear() {
429
- docToOrd.clear();
430
- recycledOrds.clear();
431
- ordToDoc.length = 0;
432
- partitionOf = new Int32Array(0);
433
- unknownPartitions = 0;
434
- mags = new Float64Array(0);
435
- if (!simd) {
436
- arena = new Float32Array(0);
437
- }
438
- dimension = 0;
439
- capacity = 0;
440
- liveCount = 0;
441
- scratchByteLength = 0;
442
- scratchFloatLength = 0;
443
- },
444
- getOrdinal(docId) {
445
- return docToOrd.get(docId);
446
- },
447
- docIdForOrdinal(ordinal) {
448
- return ordToDoc[ordinal];
449
- },
450
- entryForOrdinal(ordinal) {
451
- if (ordinal < 0 || ordinal >= ordToDoc.length || ordToDoc[ordinal] === void 0) return void 0;
452
- return entryAt(ordinal);
453
- },
454
- distanceByOrdinal(ordA, ordB, metric) {
455
- if (ordToDoc[ordA] === void 0 || ordToDoc[ordB] === void 0) return Number.POSITIVE_INFINITY;
456
- if (simd) {
457
- const byteA = scratchByteLength + ordA * dimension * 4;
458
- const byteB = scratchByteLength + ordB * dimension * 4;
459
- return arenaFloat32Distance(simd, byteA, byteB, dimension, metric, mags[ordA], mags[ordB]);
460
- }
461
- const baseA = scratchFloatLength + ordA * dimension;
462
- const baseB = scratchFloatLength + ordB * dimension;
463
- const a = arena.subarray(baseA, baseA + dimension);
464
- const b = arena.subarray(baseB, baseB + dimension);
465
- switch (metric) {
466
- case "cosine":
467
- return 1 - cosineSimilarityWithMagnitudes(a, b, mags[ordA], mags[ordB]);
468
- case "dotProduct":
469
- return -dotProduct(a, b);
470
- case "euclidean":
471
- return euclideanDistance(a, b);
472
- }
473
- },
474
- prepareQueryArena(query) {
475
- if (!simd || dimension === 0 || query.length !== dimension) return null;
476
- arena.set(query, 0);
477
- return { magnitude: simd.magnitude(0, dimension) };
478
- },
479
- distanceFromArena(prepared, ordinal, metric) {
480
- if (!simd || ordinal < 0 || ordinal >= ordToDoc.length || ordToDoc[ordinal] === void 0) {
481
- return Number.POSITIVE_INFINITY;
482
- }
483
- const byteOffset = scratchByteLength + ordinal * dimension * 4;
484
- return arenaFloat32Distance(simd, 0, byteOffset, dimension, metric, prepared.magnitude, mags[ordinal]);
485
- },
486
- exportSnapshot() {
487
- const slots = ordToDoc.length;
488
- const vectors = new Float32Array(slots * dimension);
489
- if (slots > 0 && dimension > 0) {
490
- vectors.set(arena.subarray(scratchFloatLength, scratchFloatLength + slots * dimension));
491
- }
492
- const magnitudes = new Float64Array(slots);
493
- magnitudes.set(mags.subarray(0, Math.min(slots, mags.length)));
494
- const docIds = new Array(slots);
495
- for (let ord = 0; ord < slots; ord++) {
496
- docIds[ord] = ordToDoc[ord] ?? null;
497
- }
498
- return { dimension, slots, vectors, magnitudes, docIds };
499
- },
500
- copySnapshotInto(vectors, magnitudes) {
501
- const slots = ordToDoc.length;
502
- if (slots > 0 && dimension > 0) {
503
- vectors.set(arena.subarray(scratchFloatLength, scratchFloatLength + slots * dimension));
504
- }
505
- magnitudes.set(mags.subarray(0, Math.min(slots, mags.length)));
506
- },
507
- restoreSnapshot(snapshot) {
508
- docToOrd.clear();
509
- recycledOrds.clear();
510
- ordToDoc.length = 0;
511
- partitionOf = new Int32Array(0);
512
- unknownPartitions = 0;
513
- liveCount = 0;
514
- capacity = 0;
515
- dimension = 0;
516
- scratchByteLength = 0;
517
- scratchFloatLength = 0;
518
- if (!simd) {
519
- arena = new Float32Array(0);
520
- }
521
- if (snapshot.dimension === 0 || snapshot.slots === 0) return;
522
- initStorage(snapshot.dimension);
523
- ensureCapacity2(snapshot.slots);
524
- arena.set(snapshot.vectors.subarray(0, snapshot.slots * snapshot.dimension), scratchFloatLength);
525
- mags.set(snapshot.magnitudes.subarray(0, Math.min(snapshot.slots, mags.length)));
526
- ensurePartitionSlots(snapshot.slots);
527
- for (let ord = 0; ord < snapshot.slots; ord++) {
528
- const docId = snapshot.docIds[ord];
529
- ordToDoc.push(docId ?? void 0);
530
- if (docId === null || docId === void 0) continue;
531
- docToOrd.set(docId, ord);
532
- unknownPartitions += 1;
533
- liveCount++;
534
- }
535
- },
536
- estimateMemory(dimension2) {
537
- const count = liveCount;
538
- if (count === 0) return 0;
539
- const MAP_OVERHEAD = 64;
540
- const MAP_ENTRY = 72;
541
- const AVG_DOCID_BYTES = 56;
542
- const MAGNITUDE_BYTES = 8;
543
- const ORDINAL_SLOT = 16;
544
- const perEntry = MAP_ENTRY + AVG_DOCID_BYTES + MAGNITUDE_BYTES + ORDINAL_SLOT;
545
- return MAP_OVERHEAD + count * (perEntry + dimension2 * 4);
546
- }
547
- };
548
- }
549
-
550
- // src/vector/hnsw/adjacency.ts
551
- var ABSENT = -1;
552
- function createAdjacency(m, mMax0) {
553
- return {
554
- m,
555
- mMax0,
556
- level0Stride: mMax0 + 2,
557
- upperStride: m + 2,
558
- capacity: 0,
559
- slots: 0,
560
- nodeLevels: new Int8Array(0),
561
- level0: new Int32Array(0),
562
- upperBase: new Int32Array(0),
563
- upper: new Int32Array(0),
564
- upperUsed: 0,
565
- freeBlocks: Array.from({ length: MAX_LAYER_CAP + 1 }, () => [])
566
- };
567
- }
568
- function ensureAdjacencyCapacity(adj, needed) {
569
- if (needed <= adj.capacity) return;
570
- let newCapacity = adj.capacity === 0 ? 16 : adj.capacity;
571
- while (newCapacity < needed) newCapacity *= 2;
572
- const nextLevels = new Int8Array(newCapacity).fill(ABSENT);
573
- nextLevels.set(adj.nodeLevels);
574
- adj.nodeLevels = nextLevels;
575
- const nextLevel0 = new Int32Array(newCapacity * adj.level0Stride);
576
- nextLevel0.set(adj.level0);
577
- adj.level0 = nextLevel0;
578
- const nextUpperBase = new Int32Array(newCapacity).fill(ABSENT);
579
- nextUpperBase.set(adj.upperBase);
580
- adj.upperBase = nextUpperBase;
581
- adj.capacity = newCapacity;
582
- }
583
- function allocateUpperBlock(adj, levels) {
584
- const reusable = adj.freeBlocks[levels].pop();
585
- const size = levels * adj.upperStride;
586
- if (reusable !== void 0) {
587
- adj.upper.fill(0, reusable, reusable + size);
588
- return reusable;
589
- }
590
- if (adj.upperUsed + size > adj.upper.length) {
591
- const required = adj.upperUsed + size;
592
- let nextLength = adj.upper.length === 0 ? INITIAL_UPPER_SLOTS * adj.upperStride : adj.upper.length;
593
- while (nextLength < required) nextLength *= 2;
594
- const nextUpper = new Int32Array(nextLength);
595
- nextUpper.set(adj.upper);
596
- adj.upper = nextUpper;
597
- }
598
- const offset = adj.upperUsed;
599
- adj.upperUsed += size;
600
- adj.upper.fill(0, offset, offset + size);
601
- return offset;
602
- }
603
- function createNode(adj, ord, maxLayer) {
604
- ensureAdjacencyCapacity(adj, ord + 1);
605
- if (ord >= adj.slots) adj.slots = ord + 1;
606
- const existingLevel = adj.nodeLevels[ord];
607
- const existingBase = adj.upperBase[ord];
608
- if (existingLevel >= 1 && existingBase !== ABSENT) {
609
- adj.freeBlocks[existingLevel].push(existingBase);
610
- }
611
- adj.nodeLevels[ord] = maxLayer;
612
- adj.level0[ord * adj.level0Stride] = 0;
613
- adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) : ABSENT;
614
- }
615
- function deleteNode(adj, ord) {
616
- if (ord < 0 || ord >= adj.capacity) return;
617
- const level = adj.nodeLevels[ord];
618
- const base = adj.upperBase[ord];
619
- if (level >= 1 && base !== ABSENT) {
620
- adj.freeBlocks[level].push(base);
621
- }
622
- adj.nodeLevels[ord] = ABSENT;
623
- adj.upperBase[ord] = ABSENT;
624
- adj.level0[ord * adj.level0Stride] = 0;
625
- }
626
- function resetAdjacency(adj) {
627
- adj.nodeLevels = new Int8Array(0);
628
- adj.level0 = new Int32Array(0);
629
- adj.upperBase = new Int32Array(0);
630
- adj.upper = new Int32Array(0);
631
- adj.upperUsed = 0;
632
- adj.capacity = 0;
633
- adj.slots = 0;
634
- for (const list of adj.freeBlocks) list.length = 0;
635
- }
636
- function nodeLevel(adj, ord) {
637
- if (ord < 0 || ord >= adj.capacity) return ABSENT;
638
- return adj.nodeLevels[ord];
639
- }
640
- function hasNode(adj, ord) {
641
- return nodeLevel(adj, ord) !== ABSENT;
642
- }
643
- function layerArray(adj, layer) {
644
- return layer === 0 ? adj.level0 : adj.upper;
645
- }
646
- function layerBase(adj, ord, layer) {
647
- const level = nodeLevel(adj, ord);
648
- if (level === ABSENT || layer > level) return ABSENT;
649
- if (layer === 0) return ord * adj.level0Stride;
650
- const base = adj.upperBase[ord];
651
- if (base === ABSENT) return ABSENT;
652
- return base + (layer - 1) * adj.upperStride;
653
- }
654
- function neighborCount(adj, ord, layer) {
655
- const base = layerBase(adj, ord, layer);
656
- if (base === ABSENT) return 0;
657
- return layerArray(adj, layer)[base];
658
- }
659
- function collectNeighbors(adj, ord, layer) {
660
- const base = layerBase(adj, ord, layer);
661
- if (base === ABSENT) return [];
662
- const array = layerArray(adj, layer);
663
- const count = array[base];
664
- const collected = new Array(count);
665
- for (let i = 0; i < count; i++) {
666
- collected[i] = array[base + i + 1];
667
- }
668
- return collected;
669
- }
670
- function addNeighbor(adj, ord, layer, neighborOrd) {
671
- const base = layerBase(adj, ord, layer);
672
- if (base === ABSENT) return;
673
- const array = layerArray(adj, layer);
674
- const count = array[base];
675
- for (let i = 1; i <= count; i++) {
676
- if (array[base + i] === neighborOrd) return;
677
- }
678
- const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
679
- if (count + 1 > stride - 1) return;
680
- array[base + count + 1] = neighborOrd;
681
- array[base] = count + 1;
682
- }
683
- function removeNeighbor(adj, ord, layer, neighborOrd) {
684
- const base = layerBase(adj, ord, layer);
685
- if (base === ABSENT) return;
686
- const array = layerArray(adj, layer);
687
- const count = array[base];
688
- for (let i = 1; i <= count; i++) {
689
- if (array[base + i] !== neighborOrd) continue;
690
- for (let j = i; j < count; j++) {
691
- array[base + j] = array[base + j + 1];
692
- }
693
- array[base] = count - 1;
694
- return;
695
- }
696
- }
697
- function replaceNeighbors(adj, ord, layer, neighbors, count) {
698
- const base = layerBase(adj, ord, layer);
699
- if (base === ABSENT) return;
700
- const array = layerArray(adj, layer);
701
- const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
702
- const limit = Math.min(count, stride - 1);
703
- for (let i = 0; i < limit; i++) {
704
- array[base + i + 1] = neighbors[i];
705
- }
706
- array[base] = limit;
707
- }
708
- function exportAdjacency(adj) {
709
- return {
710
- m: adj.m,
711
- mMax0: adj.mMax0,
712
- capacity: adj.capacity,
713
- slots: adj.slots,
714
- upperUsed: adj.upperUsed,
715
- nodeLevels: adj.nodeLevels.slice(),
716
- level0: adj.level0.slice(),
717
- upperBase: adj.upperBase.slice(),
718
- upper: adj.upper.slice(0, adj.upperUsed)
719
- };
720
- }
721
- function importAdjacency(snapshot) {
722
- const adj = createAdjacency(snapshot.m, snapshot.mMax0);
723
- adj.capacity = snapshot.capacity;
724
- adj.slots = snapshot.slots;
725
- adj.upperUsed = snapshot.upperUsed;
726
- adj.nodeLevels = snapshot.nodeLevels;
727
- adj.level0 = snapshot.level0;
728
- adj.upperBase = snapshot.upperBase;
729
- adj.upper = snapshot.upper;
730
- return adj;
731
- }
732
- function estimateAdjacencyBytes(adj) {
733
- return adj.nodeLevels.byteLength + adj.level0.byteLength + adj.upperBase.byteLength + adj.upper.byteLength;
734
- }
735
-
736
- // src/vector/hnsw/shared.ts
737
- function ensureCapacity(state, needed) {
738
- ensureAdjacencyCapacity(state.adjacency, needed);
739
- if (needed <= state.capacity) return;
740
- let newCap = state.capacity === 0 ? 16 : state.capacity;
741
- while (newCap < needed) newCap *= 2;
742
- const nextTombstones = new Uint8Array(newCap);
743
- nextTombstones.set(state.tombstones);
744
- state.tombstones = nextTombstones;
745
- state.visited = new Uint32Array(newCap);
746
- state.visitStamp = 0;
747
- state.capacity = newCap;
748
- }
749
- function nextVisitStamp(state) {
750
- state.visitStamp++;
751
- if (state.visitStamp === 4294967295) {
752
- state.visited.fill(0);
753
- state.visitStamp = 1;
754
- }
755
- return state.visitStamp;
756
- }
757
- function nodeExists(state, ord) {
758
- return hasNode(state.adjacency, ord);
759
- }
760
- function nodeMaxLayer(state, ord) {
761
- return nodeLevel(state.adjacency, ord);
762
- }
763
- function toDistance(a, b, magA, magB, metric) {
764
- switch (metric) {
765
- case "cosine":
766
- return 1 - cosineSimilarityWithMagnitudes(a, b, magA, magB);
767
- case "dotProduct":
768
- return -dotProduct(a, b);
769
- case "euclidean":
770
- return euclideanDistance(a, b);
771
- }
772
- }
773
- function toScore(distance, metric) {
774
- switch (metric) {
775
- case "cosine":
776
- return 1 - distance;
777
- case "dotProduct":
778
- return -distance;
779
- case "euclidean":
780
- return 1 / (1 + distance);
781
- }
782
- }
783
- function randomLevel(mL) {
784
- let u = Math.random();
785
- if (u === 0) u = Number.MIN_VALUE;
786
- return Math.min(Math.floor(-Math.log(u) * mL), MAX_LAYER_CAP);
787
- }
788
- function entryForOrd(state, ord) {
789
- return state.store.entryForOrdinal(ord);
790
- }
791
- function nodeDistanceByOrd(state, aOrd, bOrd, metric) {
792
- return state.store.distanceByOrdinal(aOrd, bOrd, metric);
793
- }
794
- function queryDistanceByOrd(state, qVec, qMag, ord, metric) {
795
- const entry = state.store.entryForOrdinal(ord);
796
- if (!entry) return Number.POSITIVE_INFINITY;
797
- return toDistance(qVec, entry.vector, qMag, entry.magnitude, metric);
798
- }
799
- function maxConns(state, layer) {
800
- return layer === 0 ? state.Mmax0 : state.M;
801
- }
802
- function addConnection(connections, ord) {
803
- for (let i = 0; i < connections.length; i++) {
804
- if (connections[i] === ord) return;
805
- }
806
- connections.push(ord);
807
- }
808
-
809
- // src/vector/hnsw/workspace.ts
810
- function createList(capacity) {
811
- return { ords: new Int32Array(capacity), distances: new Float64Array(capacity), size: 0 };
812
- }
813
- function createHeap(greatestFirst) {
814
- return { ...createList(INITIAL_LIST_CAPACITY), greatestFirst, topOrd: -1, topDistance: 0 };
815
- }
816
- function createHNSWWorkspace() {
817
- return {
818
- frontier: createHeap(false),
819
- found: createHeap(true),
820
- entryPoints: new Int32Array(INITIAL_LIST_CAPACITY),
821
- entryPointCount: 0,
822
- traversal: createList(INITIAL_LIST_CAPACITY),
823
- working: createList(INITIAL_LIST_CAPACITY),
824
- insertSelection: createList(INITIAL_LIST_CAPACITY),
825
- pruneCandidates: createList(INITIAL_LIST_CAPACITY),
826
- pruneSelection: createList(INITIAL_LIST_CAPACITY),
827
- repairCandidates: createList(INITIAL_LIST_CAPACITY),
828
- repairSelection: createList(INITIAL_LIST_CAPACITY)
829
- };
830
- }
831
- function ensureListCapacity(list, needed) {
832
- if (needed <= list.ords.length) return;
833
- let capacity = list.ords.length === 0 ? INITIAL_LIST_CAPACITY : list.ords.length;
834
- while (capacity < needed) capacity *= 2;
835
- const ords = new Int32Array(capacity);
836
- ords.set(list.ords);
837
- list.ords = ords;
838
- const distances = new Float64Array(capacity);
839
- distances.set(list.distances);
840
- list.distances = distances;
841
- }
842
- function appendToList(list, ord, distance) {
843
- ensureListCapacity(list, list.size + 1);
844
- list.ords[list.size] = ord;
845
- list.distances[list.size] = distance;
846
- list.size += 1;
847
- }
848
- function copyList(source, target) {
849
- ensureListCapacity(target, source.size);
850
- target.ords.set(source.ords.subarray(0, source.size));
851
- target.distances.set(source.distances.subarray(0, source.size));
852
- target.size = source.size;
853
- }
854
- function sortListByDistance(list) {
855
- const { ords, distances } = list;
856
- for (let i = 1; i < list.size; i++) {
857
- const ord = ords[i];
858
- const distance = distances[i];
859
- let j = i - 1;
860
- while (j >= 0 && distances[j] > distance) {
861
- distances[j + 1] = distances[j];
862
- ords[j + 1] = ords[j];
863
- j -= 1;
864
- }
865
- distances[j + 1] = distance;
866
- ords[j + 1] = ord;
867
- }
868
- }
869
- function heapPrecedes(heap, a, b) {
870
- return heap.greatestFirst ? heap.distances[a] > heap.distances[b] : heap.distances[a] < heap.distances[b];
871
- }
872
- function swapHeapEntries(heap, a, b) {
873
- const ord = heap.ords[a];
874
- heap.ords[a] = heap.ords[b];
875
- heap.ords[b] = ord;
876
- const distance = heap.distances[a];
877
- heap.distances[a] = heap.distances[b];
878
- heap.distances[b] = distance;
879
- }
880
- function pushHeap(heap, ord, distance) {
881
- appendToList(heap, ord, distance);
882
- let index = heap.size - 1;
883
- while (index > 0) {
884
- const parent = index - 1 >> 1;
885
- if (!heapPrecedes(heap, index, parent)) break;
886
- swapHeapEntries(heap, index, parent);
887
- index = parent;
888
- }
889
- }
890
- function popHeap(heap) {
891
- if (heap.size === 0) return false;
892
- heap.topOrd = heap.ords[0];
893
- heap.topDistance = heap.distances[0];
894
- heap.size -= 1;
895
- if (heap.size > 0) {
896
- heap.ords[0] = heap.ords[heap.size];
897
- heap.distances[0] = heap.distances[heap.size];
898
- let index = 0;
899
- for (; ; ) {
900
- const left = 2 * index + 1;
901
- const right = left + 1;
902
- let first = index;
903
- if (left < heap.size && heapPrecedes(heap, left, first)) first = left;
904
- if (right < heap.size && heapPrecedes(heap, right, first)) first = right;
905
- if (first === index) break;
906
- swapHeapEntries(heap, index, first);
907
- index = first;
908
- }
909
- }
910
- return true;
911
- }
912
- function resetHeap(heap) {
913
- heap.size = 0;
914
- }
915
- function drainHeapNearestFirst(heap, list) {
916
- const count = heap.size;
917
- ensureListCapacity(list, count);
918
- list.size = count;
919
- for (let i = count - 1; i >= 0; i--) {
920
- if (!popHeap(heap)) break;
921
- list.ords[i] = heap.topOrd;
922
- list.distances[i] = heap.topDistance;
923
- }
924
- }
925
- function setSingleEntryPoint(workspace, ord) {
926
- workspace.entryPoints[0] = ord;
927
- workspace.entryPointCount = 1;
928
- }
929
- function setEntryPointsFromList(workspace, list) {
930
- if (list.size > workspace.entryPoints.length) {
931
- let capacity = workspace.entryPoints.length;
932
- while (capacity < list.size) capacity *= 2;
933
- workspace.entryPoints = new Int32Array(capacity);
934
- }
935
- workspace.entryPoints.set(list.ords.subarray(0, list.size));
936
- workspace.entryPointCount = list.size;
937
- }
938
-
939
- // src/vector/hnsw/graph-ops.ts
940
- function searchLayer(state, qVec, qMag, ef, layer, metric, skipTombstones, distFn, results) {
941
- const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
942
- const workspace = state.workspace;
943
- const frontier = workspace.frontier;
944
- const found = workspace.found;
945
- const adjacency = state.adjacency;
946
- const neighbors = layerArray(adjacency, layer);
947
- const visited = state.visited;
948
- const stamp = nextVisitStamp(state);
949
- resetHeap(frontier);
950
- resetHeap(found);
951
- let furthestDist = Number.POSITIVE_INFINITY;
952
- for (let i = 0; i < workspace.entryPointCount; i++) {
953
- const epOrd = workspace.entryPoints[i];
954
- if (visited[epOrd] === stamp) continue;
955
- visited[epOrd] = stamp;
956
- if (!nodeExists(state, epOrd)) continue;
957
- if (skipTombstones && state.tombstones[epOrd] === 1) continue;
958
- const dist = getDistance(epOrd);
959
- if (dist === Number.POSITIVE_INFINITY) continue;
960
- pushHeap(frontier, epOrd, dist);
961
- pushHeap(found, epOrd, dist);
962
- if (found.size > ef) {
963
- popHeap(found);
964
- }
965
- }
966
- if (found.size > 0) furthestDist = found.distances[0];
967
- while (popHeap(frontier)) {
968
- if (frontier.topDistance > furthestDist) break;
969
- const base = layerBase(adjacency, frontier.topOrd, layer);
970
- if (base === -1) continue;
971
- const count = neighbors[base];
972
- for (let i = 1; i <= count; i++) {
973
- const neighborOrd = neighbors[base + i];
974
- if (visited[neighborOrd] === stamp) continue;
975
- visited[neighborOrd] = stamp;
976
- if (skipTombstones && state.tombstones[neighborOrd] === 1) continue;
977
- if (!nodeExists(state, neighborOrd)) continue;
978
- const dist = getDistance(neighborOrd);
979
- if (dist === Number.POSITIVE_INFINITY) continue;
980
- if (dist < furthestDist || found.size < ef) {
981
- pushHeap(frontier, neighborOrd, dist);
982
- pushHeap(found, neighborOrd, dist);
983
- if (found.size > ef) {
984
- popHeap(found);
985
- }
986
- furthestDist = found.distances[0];
987
- }
988
- }
989
- }
990
- drainHeapNearestFirst(found, results);
991
- }
992
- function selectNeighborsHeuristic(state, candidates, maxConnections, metric, selected) {
993
- const working = state.workspace.working;
994
- copyList(candidates, working);
995
- sortListByDistance(working);
996
- ensureListCapacity(selected, Math.min(maxConnections, working.size));
997
- selected.size = 0;
998
- for (let i = 0; i < working.size; i++) {
999
- if (selected.size >= maxConnections) break;
1000
- const candidateOrd = working.ords[i];
1001
- const candidateDistance = working.distances[i];
1002
- let accepted = true;
1003
- for (let s = 0; s < selected.size; s++) {
1004
- const distBetween = nodeDistanceByOrd(state, candidateOrd, selected.ords[s], metric);
1005
- if (candidateDistance >= distBetween) {
1006
- accepted = false;
1007
- break;
1008
- }
1009
- }
1010
- if (!accepted) continue;
1011
- selected.ords[selected.size] = candidateOrd;
1012
- selected.distances[selected.size] = candidateDistance;
1013
- selected.size += 1;
1014
- }
1015
- }
1016
- function pruneConnections(state, ord, layer, metric) {
1017
- const adjacency = state.adjacency;
1018
- const base = layerBase(adjacency, ord, layer);
1019
- if (base === -1) return;
1020
- const mc = maxConns(state, layer);
1021
- const neighbors = layerArray(adjacency, layer);
1022
- const count = neighbors[base];
1023
- if (count <= mc) return;
1024
- const candidates = state.workspace.pruneCandidates;
1025
- candidates.size = 0;
1026
- ensureListCapacity(candidates, count);
1027
- for (let i = 1; i <= count; i++) {
1028
- const connOrd = neighbors[base + i];
1029
- const dist = nodeDistanceByOrd(state, ord, connOrd, metric);
1030
- if (dist === Number.POSITIVE_INFINITY) continue;
1031
- candidates.ords[candidates.size] = connOrd;
1032
- candidates.distances[candidates.size] = dist;
1033
- candidates.size += 1;
1034
- }
1035
- const kept = state.workspace.pruneSelection;
1036
- selectNeighborsHeuristic(state, candidates, mc, metric, kept);
1037
- replaceNeighbors(adjacency, ord, layer, kept.ords, kept.size);
1038
- }
1039
-
1040
- // src/vector/hnsw/search.ts
1041
- var ABSENT_DOCUMENT_RANK = 4294967295;
1042
- function collectHits(state, query, k, searchMetric, minSimilarity, filter, efSearch, hasDocument) {
1043
- if (query.length !== state.dimension) {
1044
- throw new Error(`Query dimension mismatch: expected ${state.dimension}, got ${query.length}`);
1045
- }
1046
- const liveSize = state.nodeCount - state.tombstoneCount;
1047
- if (state.entryPointOrd === -1 || liveSize === 0) {
1048
- return [];
1049
- }
1050
- const useQuantized = state.quantizer?.isCalibrated() === true && state.quantizer.size > 0;
1051
- const defaultEf = 50;
1052
- let ef = Math.max(efSearch ?? defaultEf, k);
1053
- if (filter && filter.count < liveSize) {
1054
- const selectivity = filter.count / liveSize;
1055
- ef = Math.max(ef, Math.ceil(k / Math.max(selectivity, 0.01)));
1056
- ef = Math.min(ef, liveSize);
1057
- }
1058
- const store = state.store;
1059
- const arenaQuery = store.prepareQueryArena(query);
1060
- const qMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
1061
- let quantizedDistFn;
1062
- if (useQuantized && state.quantizer) {
1063
- const q = state.quantizer;
1064
- const metric = searchMetric;
1065
- const quantizedArenaQuery = q.prepareQueryArena(query);
1066
- if (quantizedArenaQuery) {
1067
- quantizedDistFn = (ord) => q.distanceFromArena(quantizedArenaQuery, ord, metric);
1068
- } else {
1069
- const prepared = q.prepareQuery(query);
1070
- if (prepared) {
1071
- quantizedDistFn = (ord) => q.distanceFromPreparedByOrdinal(prepared, ord, metric);
1072
- }
1073
- }
1074
- }
1075
- let distFn = quantizedDistFn;
1076
- if (!distFn && arenaQuery) {
1077
- const metric = searchMetric;
1078
- distFn = (ord) => store.distanceFromArena(arenaQuery, ord, metric);
1079
- }
1080
- const workspace = state.workspace;
1081
- const candidates = workspace.traversal;
1082
- setSingleEntryPoint(workspace, state.entryPointOrd);
1083
- for (let layer = state.topLayer; layer >= 1; layer--) {
1084
- searchLayer(state, query, qMag, 1, layer, searchMetric, true, distFn, candidates);
1085
- if (candidates.size > 0) {
1086
- setSingleEntryPoint(workspace, candidates.ords[0]);
1087
- }
1088
- }
1089
- searchLayer(state, query, qMag, ef, 0, searchMetric, true, distFn, candidates);
1090
- if (useQuantized) {
1091
- return rerankWithFullPrecision(
1092
- state,
1093
- candidates,
1094
- query,
1095
- qMag,
1096
- arenaQuery,
1097
- k,
1098
- searchMetric,
1099
- minSimilarity,
1100
- filter,
1101
- hasDocument
1102
- );
1103
- }
1104
- const hits = [];
1105
- for (let i = 0; i < candidates.size; i++) {
1106
- const ord = candidates.ords[i];
1107
- if (filter && !ordinalFilterHas(filter, ord)) continue;
1108
- const score = toScore(candidates.distances[i], searchMetric);
1109
- if (score < minSimilarity) continue;
1110
- if (!hasDocument(ord)) continue;
1111
- hits.push({ ord, score });
1112
- }
1113
- return hits;
1114
- }
1115
- function rerankWithFullPrecision(state, candidates, query, qMag, arenaQuery, k, metric, minSimilarity, filter, hasDocument) {
1116
- const reranked = [];
1117
- const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR);
1118
- for (let i = 0; i < candidates.size; i++) {
1119
- const ord = candidates.ords[i];
1120
- if (filter && !ordinalFilterHas(filter, ord)) continue;
1121
- let fullDistance;
1122
- if (arenaQuery) {
1123
- fullDistance = state.store.distanceFromArena(arenaQuery, ord, metric);
1124
- if (fullDistance === Number.POSITIVE_INFINITY) continue;
1125
- } else {
1126
- const entry = entryForOrd(state, ord);
1127
- if (!entry) continue;
1128
- fullDistance = toDistance(query, entry.vector, qMag, entry.magnitude, metric);
1129
- }
1130
- const score = toScore(fullDistance, metric);
1131
- if (score < minSimilarity) continue;
1132
- if (!hasDocument(ord)) continue;
1133
- reranked.push({ ord, score });
1134
- if (reranked.length >= rerankLimit) break;
1135
- }
1136
- return reranked;
1137
- }
1138
- function search(state, query, k, searchMetric, minSimilarity, filter, efSearch) {
1139
- const hasDocument = (ord) => state.store.docIdForOrdinal(ord) !== void 0;
1140
- const hits = collectHits(state, query, k, searchMetric, minSimilarity, filter, efSearch, hasDocument);
1141
- const results = [];
1142
- for (const hit of hits) {
1143
- const docId = state.store.docIdForOrdinal(hit.ord);
1144
- if (docId === void 0) continue;
1145
- results.push({
1146
- docId,
1147
- score: hit.score,
1148
- termFrequencies: {},
1149
- fieldLengths: {},
1150
- idf: {}
1151
- });
1152
- }
1153
- results.sort((a, b) => b.score - a.score || compareCodePoints(a.docId, b.docId));
1154
- return results.slice(0, k);
1155
- }
1156
- function searchOrdinals(state, query, k, searchMetric, minSimilarity, rankByOrdinal, filter, efSearch) {
1157
- const hasDocument = (ord) => ord < rankByOrdinal.length && rankByOrdinal[ord] !== ABSENT_DOCUMENT_RANK;
1158
- const hits = collectHits(state, query, k, searchMetric, minSimilarity, filter, efSearch, hasDocument);
1159
- hits.sort((a, b) => b.score - a.score || rankByOrdinal[a.ord] - rankByOrdinal[b.ord]);
1160
- return hits.slice(0, k);
1161
- }
1162
-
1163
- // src/vector/hnsw/mutation.ts
1164
- function clearTombstone(state, ord) {
1165
- if (state.tombstones[ord] === 1) {
1166
- state.tombstones[ord] = 0;
1167
- state.tombstoneCount--;
1168
- }
1169
- }
1170
- function highestNode(state, includeTombstoned) {
1171
- let bestOrd = -1;
1172
- let bestLayer = -1;
1173
- for (let ord = 0; ord < state.adjacency.slots; ord++) {
1174
- const level = nodeMaxLayer(state, ord);
1175
- if (level === -1) continue;
1176
- if (!includeTombstoned && state.tombstones[ord] === 1) continue;
1177
- if (level > bestLayer) {
1178
- bestLayer = level;
1179
- bestOrd = ord;
1180
- }
1181
- }
1182
- return bestOrd;
1183
- }
1184
- function setEntryPoint(state, ord) {
1185
- state.entryPointOrd = ord;
1186
- state.topLayer = ord === -1 ? -1 : nodeMaxLayer(state, ord);
1187
- }
1188
- function insertNode(state, docId) {
1189
- const ord = state.store.getOrdinal(docId);
1190
- if (ord === void 0) {
1191
- throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
1192
- }
1193
- const entry = state.store.entryForOrdinal(ord);
1194
- if (!entry) {
1195
- throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
1196
- }
1197
- if (entry.vector.length !== state.dimension) {
1198
- throw new Error(`Vector dimension mismatch: expected ${state.dimension}, got ${entry.vector.length}`);
1199
- }
1200
- ensureCapacity(state, ord + 1);
1201
- if (nodeExists(state, ord)) {
1202
- removeNodeEager(state, ord);
1203
- }
1204
- clearTombstone(state, ord);
1205
- const l = randomLevel(state.mL);
1206
- createNode(state.adjacency, ord, l);
1207
- state.nodeCount++;
1208
- if (state.entryPointOrd === -1) {
1209
- state.entryPointOrd = ord;
1210
- state.topLayer = l;
1211
- return;
1212
- }
1213
- const metric = state.buildMetric;
1214
- const workspace = state.workspace;
1215
- const candidates = workspace.traversal;
1216
- const selected = workspace.insertSelection;
1217
- const insertDistFn = (candOrd) => nodeDistanceByOrd(state, ord, candOrd, metric);
1218
- setSingleEntryPoint(workspace, state.entryPointOrd);
1219
- for (let layer = state.topLayer; layer > l; layer--) {
1220
- searchLayer(state, entry.vector, entry.magnitude, 1, layer, metric, false, insertDistFn, candidates);
1221
- if (candidates.size > 0) {
1222
- setSingleEntryPoint(workspace, candidates.ords[0]);
1223
- }
1224
- }
1225
- for (let layer = Math.min(l, state.topLayer); layer >= 0; layer--) {
1226
- searchLayer(state, entry.vector, entry.magnitude, state.efCons, layer, metric, false, insertDistFn, candidates);
1227
- selectNeighborsHeuristic(state, candidates, maxConns(state, layer), metric, selected);
1228
- for (let i = 0; i < selected.size; i++) {
1229
- const neighborOrd = selected.ords[i];
1230
- addNeighbor(state.adjacency, ord, layer, neighborOrd);
1231
- if (layer <= nodeMaxLayer(state, neighborOrd)) {
1232
- addNeighbor(state.adjacency, neighborOrd, layer, ord);
1233
- pruneConnections(state, neighborOrd, layer, metric);
1234
- }
1235
- }
1236
- if (candidates.size > 0) {
1237
- setEntryPointsFromList(workspace, candidates);
1238
- }
1239
- }
1240
- if (l > state.topLayer) {
1241
- state.entryPointOrd = ord;
1242
- state.topLayer = l;
1243
- }
1244
- }
1245
- function removeNodeEager(state, ord, excludeOrds) {
1246
- const maxLayer = nodeMaxLayer(state, ord);
1247
- if (maxLayer === -1) return;
1248
- const metric = state.buildMetric;
1249
- for (let layer = 0; layer <= maxLayer; layer++) {
1250
- const formerNeighbors = collectNeighbors(state.adjacency, ord, layer);
1251
- for (const neighborOrd of formerNeighbors) {
1252
- if (layer <= nodeMaxLayer(state, neighborOrd)) {
1253
- removeNeighbor(state.adjacency, neighborOrd, layer, ord);
1254
- }
1255
- }
1256
- for (const neighborOrd of formerNeighbors) {
1257
- if (layer > nodeMaxLayer(state, neighborOrd)) continue;
1258
- const mc = maxConns(state, layer);
1259
- if (neighborCount(state.adjacency, neighborOrd, layer) >= mc) continue;
1260
- const candidateOrds = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
1261
- for (const otherOrd of formerNeighbors) {
1262
- if (otherOrd !== neighborOrd && otherOrd !== ord) {
1263
- if (excludeOrds?.has(otherOrd)) continue;
1264
- candidateOrds.add(otherOrd);
1265
- }
1266
- }
1267
- const candidates = state.workspace.repairCandidates;
1268
- candidates.size = 0;
1269
- for (const candOrd of candidateOrds) {
1270
- const dist = nodeDistanceByOrd(state, neighborOrd, candOrd, metric);
1271
- if (dist === Number.POSITIVE_INFINITY) continue;
1272
- appendToList(candidates, candOrd, dist);
1273
- }
1274
- const selected = state.workspace.repairSelection;
1275
- selectNeighborsHeuristic(state, candidates, mc, metric, selected);
1276
- replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
1277
- for (let i = 0; i < selected.size; i++) {
1278
- const newConnOrd = selected.ords[i];
1279
- if (layer <= nodeMaxLayer(state, newConnOrd)) {
1280
- addNeighbor(state.adjacency, newConnOrd, layer, neighborOrd);
1281
- pruneConnections(state, newConnOrd, layer, metric);
1282
- }
1283
- }
1284
- }
1285
- }
1286
- deleteNode(state.adjacency, ord);
1287
- state.nodeCount--;
1288
- clearTombstone(state, ord);
1289
- if (state.entryPointOrd === ord) {
1290
- if (state.nodeCount === 0) {
1291
- setEntryPoint(state, -1);
1292
- return;
1293
- }
1294
- const live = highestNode(state, false);
1295
- setEntryPoint(state, live === -1 ? highestNode(state, true) : live);
1296
- }
1297
- }
1298
- function markTombstone(state, docId) {
1299
- const ord = state.store.getOrdinal(docId);
1300
- if (ord === void 0 || !nodeExists(state, ord)) return;
1301
- if (state.tombstones[ord] === 0) {
1302
- state.tombstones[ord] = 1;
1303
- state.tombstoneCount++;
1304
- }
1305
- if (state.entryPointOrd === ord) {
1306
- setEntryPoint(state, highestNode(state, false));
1307
- }
1308
- }
1309
- function compactTombstones(state) {
1310
- if (state.tombstoneCount === 0) return;
1311
- const tombstonedOrds = [];
1312
- for (let ord = 0; ord < state.adjacency.slots; ord++) {
1313
- if (state.tombstones[ord] === 1 && nodeExists(state, ord)) {
1314
- tombstonedOrds.push(ord);
1315
- }
1316
- }
1317
- const excl = new Set(tombstonedOrds);
1318
- for (const ord of tombstonedOrds) {
1319
- removeNodeEager(state, ord, excl);
1320
- }
1321
- }
1322
- function rebuild(state) {
1323
- if (state.tombstoneCount === 0 && state.nodeCount === 0) return;
1324
- const liveOrds = [];
1325
- for (let ord = 0; ord < state.adjacency.slots; ord++) {
1326
- if (nodeExists(state, ord) && state.tombstones[ord] !== 1) {
1327
- liveOrds.push(ord);
1328
- }
1329
- }
1330
- resetAdjacency(state.adjacency);
1331
- state.tombstones.fill(0);
1332
- state.tombstoneCount = 0;
1333
- state.nodeCount = 0;
1334
- state.entryPointOrd = -1;
1335
- state.topLayer = -1;
1336
- for (const ord of liveOrds) {
1337
- const docId = state.store.docIdForOrdinal(ord);
1338
- if (docId === void 0) continue;
1339
- if (state.store.entryForOrdinal(ord) === void 0) continue;
1340
- insertNode(state, docId);
1341
- }
1342
- }
1343
-
1344
- // src/vector/hnsw/persistence.ts
1345
- function serializeGraph(state) {
1346
- const nodeArray = [];
1347
- for (let ord = 0; ord < state.adjacency.slots; ord++) {
1348
- const maxLayer = nodeMaxLayer(state, ord);
1349
- if (maxLayer === -1) continue;
1350
- if (state.tombstones[ord] === 1) continue;
1351
- const docId = state.store.docIdForOrdinal(ord);
1352
- if (docId === void 0) continue;
1353
- const layerConns = [];
1354
- for (let layer = 0; layer <= maxLayer; layer++) {
1355
- const liveNeighbors = [];
1356
- for (const neighborOrd of collectNeighbors(state.adjacency, ord, layer)) {
1357
- if (state.tombstones[neighborOrd] === 1) continue;
1358
- const neighborDoc = state.store.docIdForOrdinal(neighborOrd);
1359
- if (neighborDoc === void 0) continue;
1360
- liveNeighbors.push(neighborDoc);
1361
- }
1362
- if (liveNeighbors.length > 0) {
1363
- layerConns.push([layer, liveNeighbors]);
1364
- }
1365
- }
1366
- nodeArray.push([docId, maxLayer, layerConns]);
1367
- }
1368
- const entryPoint = state.entryPointOrd === -1 ? null : state.store.docIdForOrdinal(state.entryPointOrd) ?? null;
1369
- return {
1370
- entryPoint,
1371
- maxLayer: state.topLayer,
1372
- m: state.M,
1373
- efConstruction: state.efCons,
1374
- metric: state.buildMetric,
1375
- nodes: nodeArray
1376
- };
1377
- }
1378
- function resolveNodes(state, data) {
1379
- const nodes = [];
1380
- let maxOrd = -1;
1381
- for (const [docId, maxLayer, layerConns] of data.nodes) {
1382
- const ord = state.store.getOrdinal(docId);
1383
- if (ord === void 0) continue;
1384
- if (state.store.entryForOrdinal(ord) === void 0) continue;
1385
- const clampedMaxLayer = Math.min(Math.max(maxLayer, 0), MAX_LAYER_CAP);
1386
- const layers = [];
1387
- for (const [layer, neighbors] of layerConns) {
1388
- if (layer < 0 || layer > clampedMaxLayer) continue;
1389
- const limit = maxConns(state, layer);
1390
- const resolved = [];
1391
- for (const neighborDoc of neighbors) {
1392
- if (resolved.length >= limit) break;
1393
- const neighborOrd = state.store.getOrdinal(neighborDoc);
1394
- if (neighborOrd === void 0) continue;
1395
- addConnection(resolved, neighborOrd);
1396
- if (neighborOrd > maxOrd) maxOrd = neighborOrd;
1397
- }
1398
- layers.push([layer, resolved]);
1399
- }
1400
- if (ord > maxOrd) maxOrd = ord;
1401
- nodes.push({ ord, maxLayer: clampedMaxLayer, layers });
1402
- }
1403
- return { nodes, maxOrd };
1404
- }
1405
- function deserializeGraph(state, data) {
1406
- resetAdjacency(state.adjacency);
1407
- state.tombstones.fill(0);
1408
- state.tombstoneCount = 0;
1409
- state.nodeCount = 0;
1410
- state.entryPointOrd = -1;
1411
- state.topLayer = -1;
1412
- const { nodes, maxOrd } = resolveNodes(state, data);
1413
- if (maxOrd >= 0) {
1414
- ensureCapacity(state, maxOrd + 1);
1415
- }
1416
- for (const node of nodes) {
1417
- createNode(state.adjacency, node.ord, node.maxLayer);
1418
- for (const [layer, neighbors] of node.layers) {
1419
- replaceNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
1420
- }
1421
- state.nodeCount++;
1422
- }
1423
- if (data.entryPoint != null && data.entryPoint !== "") {
1424
- const epOrd = state.store.getOrdinal(data.entryPoint);
1425
- if (epOrd !== void 0 && nodeExists(state, epOrd)) {
1426
- state.entryPointOrd = epOrd;
1427
- state.topLayer = Math.min(Math.max(data.maxLayer, 0), MAX_LAYER_CAP);
1428
- }
1429
- }
1430
- if (state.entryPointOrd === -1 && state.nodeCount > 0) {
1431
- let bestOrd = -1;
1432
- let bestLayer = -1;
1433
- for (let ord = 0; ord < state.adjacency.slots; ord++) {
1434
- const level = nodeMaxLayer(state, ord);
1435
- if (level === -1) continue;
1436
- if (level > bestLayer) {
1437
- bestLayer = level;
1438
- bestOrd = ord;
1439
- }
1440
- }
1441
- state.entryPointOrd = bestOrd;
1442
- state.topLayer = bestLayer;
1443
- }
1444
- }
1445
-
1446
- // src/vector/hnsw/index.ts
1447
- function createHNSWIndex(dimension, store, config, quantizer) {
1448
- const M = config?.m ?? 16;
1449
- const Mmax0 = M * 2;
1450
- const efCons = config?.efConstruction ?? 200;
1451
- const buildMetric = config?.metric ?? "cosine";
1452
- const mL = 1 / Math.log(M);
1453
- const state = {
1454
- dimension,
1455
- store,
1456
- quantizer,
1457
- M,
1458
- Mmax0,
1459
- efCons,
1460
- buildMetric,
1461
- mL,
1462
- adjacency: createAdjacency(M, Mmax0),
1463
- tombstones: new Uint8Array(0),
1464
- tombstoneCount: 0,
1465
- nodeCount: 0,
1466
- capacity: 0,
1467
- visited: new Uint32Array(0),
1468
- visitStamp: 0,
1469
- entryPointOrd: -1,
1470
- topLayer: -1,
1471
- workspace: createHNSWWorkspace()
1472
- };
1473
- function clear() {
1474
- resetAdjacency(state.adjacency);
1475
- state.tombstones = new Uint8Array(0);
1476
- state.tombstoneCount = 0;
1477
- state.nodeCount = 0;
1478
- state.capacity = 0;
1479
- state.visited = new Uint32Array(0);
1480
- state.visitStamp = 0;
1481
- state.entryPointOrd = -1;
1482
- state.topLayer = -1;
1483
- }
1484
- function* entriesIterator() {
1485
- for (let ord = 0; ord < state.adjacency.slots; ord++) {
1486
- if (!hasNode(state.adjacency, ord)) continue;
1487
- if (state.tombstones[ord] === 1) continue;
1488
- const docId = state.store.docIdForOrdinal(ord);
1489
- if (docId === void 0) continue;
1490
- const entry = state.store.entryForOrdinal(ord);
1491
- if (!entry) continue;
1492
- yield [docId, { docId, vector: entry.vector, magnitude: entry.magnitude }];
1493
- }
1494
- }
1495
- function compactionNeeded() {
1496
- if (state.nodeCount === 0) return false;
1497
- return state.tombstoneCount / state.nodeCount > COMPACTION_TOMBSTONE_RATIO || state.tombstoneCount > COMPACTION_ABSOLUTE_THRESHOLD;
1498
- }
1499
- return {
1500
- get dimension() {
1501
- return state.dimension;
1502
- },
1503
- get size() {
1504
- return state.nodeCount - state.tombstoneCount;
1505
- },
1506
- get tombstoneCount() {
1507
- return state.tombstoneCount;
1508
- },
1509
- get entryPointId() {
1510
- return state.entryPointOrd === -1 ? null : state.store.docIdForOrdinal(state.entryPointOrd) ?? null;
1511
- },
1512
- get topLayer() {
1513
- return state.topLayer;
1514
- },
1515
- get m() {
1516
- return state.M;
1517
- },
1518
- get efConstruction() {
1519
- return state.efCons;
1520
- },
1521
- get metric() {
1522
- return state.buildMetric;
1523
- },
1524
- get adjacencyBytes() {
1525
- return estimateAdjacencyBytes(state.adjacency);
1526
- },
1527
- insertNode: (docId) => insertNode(state, docId),
1528
- markTombstone: (docId) => markTombstone(state, docId),
1529
- has: (docId) => {
1530
- const ord = state.store.getOrdinal(docId);
1531
- return ord !== void 0 && hasNode(state.adjacency, ord) && state.tombstones[ord] !== 1;
1532
- },
1533
- isTombstoned: (docId) => {
1534
- const ord = state.store.getOrdinal(docId);
1535
- return ord !== void 0 && state.tombstones[ord] === 1;
1536
- },
1537
- search: (query, k, searchMetric, minSimilarity, filter, efSearch) => search(state, query, k, searchMetric, minSimilarity, filter, efSearch),
1538
- clear,
1539
- entries: entriesIterator,
1540
- compactionNeeded,
1541
- compactTombstones: () => compactTombstones(state),
1542
- rebuild: () => rebuild(state),
1543
- serialize: () => serializeGraph(state),
1544
- deserialize: (data) => deserializeGraph(state, data),
1545
- exportSnapshot: () => ({
1546
- dimension: state.dimension,
1547
- m: state.M,
1548
- efConstruction: state.efCons,
1549
- metric: state.buildMetric,
1550
- adjacency: exportAdjacency(state.adjacency),
1551
- tombstones: state.tombstones.slice(),
1552
- tombstoneCount: state.tombstoneCount,
1553
- nodeCount: state.nodeCount,
1554
- capacity: state.capacity,
1555
- entryPointOrd: state.entryPointOrd,
1556
- topLayer: state.topLayer
1557
- }),
1558
- restoreSnapshot: (snapshot) => {
1559
- state.adjacency = importAdjacency(snapshot.adjacency);
1560
- state.tombstones = snapshot.tombstones;
1561
- state.tombstoneCount = snapshot.tombstoneCount;
1562
- state.nodeCount = snapshot.nodeCount;
1563
- state.capacity = Math.max(snapshot.capacity, snapshot.adjacency.capacity);
1564
- state.visited = new Uint32Array(state.capacity);
1565
- state.visitStamp = 0;
1566
- state.entryPointOrd = snapshot.entryPointOrd;
1567
- state.topLayer = snapshot.topLayer;
1568
- }
1569
- };
1570
- }
1571
-
1572
- export { ABSENT_DOCUMENT_RANK, HNSW_BUILD_TIMEOUT_MS, MAX_WASM_PAGES, MAX_WORKER_COPIES, MAX_WORKER_DIMENSION, MAX_WORKER_VECTORS, QUANTIZATION_PADDING_FACTOR, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createArenaSimd, createHNSWIndex, createHNSWWorkspace, createOrdinalFilter, createSharedArenaSimd, createVectorStore, dotProduct, entryForOrd, euclideanDistance, importAdjacency, isSimdAvailable, magnitude, ordinalFilterHas, ordinalFilterValues, removeFromOrdinalFilter, searchOrdinals, toDistance, toScore };