@delali/narsil 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +800 -111
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +46 -2
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-BbRkoiu9.d.ts → adapters-Bo2JQg8b.d.ts} +2 -1
- package/dist/chunk-ALB35PRQ.mjs +98 -0
- package/dist/chunk-KMMJMDUA.mjs +44 -0
- package/dist/chunk-RQLXC2YS.mjs +269 -0
- package/dist/chunk-S7RJJMHI.mjs +1086 -0
- package/dist/chunk-SVDEEMO6.mjs +7023 -0
- package/dist/{chunk-I24SGLFS.mjs → chunk-TW6BXTOL.mjs} +5 -1
- package/dist/{chunk-KHL4RPC3.mjs → chunk-VWF7RKSF.mjs} +2566 -2282
- package/dist/chunk-WISGW24C.mjs +455 -0
- package/dist/config-DX--2Owg.d.ts +472 -0
- package/dist/direct-executor-KC5O5D2R.mjs +4 -0
- package/dist/distribution/coordinator/etcd.d.ts +16 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -0
- package/dist/distribution/coordinator/in-memory.d.ts +7 -0
- package/dist/distribution/coordinator/in-memory.mjs +1 -0
- package/dist/distribution.d.ts +122 -0
- package/dist/distribution.mjs +7410 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +6 -1
- package/dist/index.browser.mjs +11750 -7532
- package/dist/index.d.ts +6 -450
- package/dist/index.mjs +7 -3236
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{language-ClUGLuoC.d.ts → language-ONQ3iVDf.d.ts} +1 -0
- package/dist/languages/arabic.d.ts +1 -1
- package/dist/languages/armenian.d.ts +1 -1
- package/dist/languages/bulgarian.d.ts +1 -1
- package/dist/languages/chinese.d.ts +1 -1
- package/dist/languages/dagbani.d.ts +1 -1
- package/dist/languages/danish.d.ts +1 -1
- package/dist/languages/dutch.d.ts +1 -1
- package/dist/languages/english.d.ts +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/ewe.d.ts +1 -1
- package/dist/languages/finnish.d.ts +1 -1
- package/dist/languages/french.d.ts +1 -1
- package/dist/languages/ga.d.ts +1 -1
- package/dist/languages/german.d.ts +1 -1
- package/dist/languages/greek.d.ts +1 -1
- package/dist/languages/hausa.d.ts +1 -1
- package/dist/languages/hindi.d.ts +1 -1
- package/dist/languages/hungarian.d.ts +1 -1
- package/dist/languages/igbo.d.ts +1 -1
- package/dist/languages/indonesian.d.ts +1 -1
- package/dist/languages/irish.d.ts +1 -1
- package/dist/languages/italian.d.ts +1 -1
- package/dist/languages/japanese.d.ts +1 -1
- package/dist/languages/nepali.d.ts +1 -1
- package/dist/languages/norwegian.d.ts +1 -1
- package/dist/languages/portuguese.d.ts +1 -1
- package/dist/languages/romanian.d.ts +1 -1
- package/dist/languages/russian.d.ts +1 -1
- package/dist/languages/sanskrit.d.ts +1 -1
- package/dist/languages/serbian.d.ts +1 -1
- package/dist/languages/slovenian.d.ts +1 -1
- package/dist/languages/spanish.d.ts +1 -1
- package/dist/languages/swahili.d.ts +1 -1
- package/dist/languages/swedish.d.ts +1 -1
- package/dist/languages/tamil.d.ts +1 -1
- package/dist/languages/turkish.d.ts +1 -1
- package/dist/languages/twi.d.ts +1 -1
- package/dist/languages/ukrainian.d.ts +1 -1
- package/dist/languages/yoruba.d.ts +1 -1
- package/dist/languages/zulu.d.ts +1 -1
- package/dist/narsil-B3jKyXCr.d.ts +72 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +48 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +59 -0
- package/dist/schema-C3VhNvfw.d.ts +60 -0
- package/dist/serialization/crc32-worker.d.ts +20 -0
- package/dist/serialization/crc32-worker.mjs +61 -0
- package/dist/server.d.ts +218 -0
- package/dist/server.mjs +1184 -0
- package/dist/types-CTVVm4GG.d.ts +73 -0
- package/dist/vector/hnsw-build-worker.mjs +1 -1
- package/dist/workers/entry.mjs +1 -1
- package/package.json +38 -2
- package/dist/chunk-R2C2EQGO.mjs +0 -53
- package/dist/chunk-VVORJOXR.mjs +0 -759
- package/dist/direct-executor-LBJLGMJ2.mjs +0 -4
package/dist/chunk-VVORJOXR.mjs
DELETED
|
@@ -1,759 +0,0 @@
|
|
|
1
|
-
// src/vector/simd-wasm-binary.ts
|
|
2
|
-
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABDgJgA39/fwF9YAJ/fwF9AwUEAAEAAAUDAQABB1YFBm1lbW9yeQIAC2RvdF9wcm9kdWN0AAAJbWFnbml0dWRlAAEac3F1YXJlZF9ldWNsaWRlYW5fZGlzdGFuY2UAAhJldWNsaWRlYW5fZGlzdGFuY2UAAwqqBASuAQQBfwF7AX0BfyACQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhBEEAIQMCQANAIAMgBk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAE/R8AIAT9HwGSIAT9HwIgBP0fA5KSIQUCQANAIAMgAk8NASAFIAAgA0ECdGoqAgAgASADQQJ0aioCAJSSIQUgA0EBaiEDDAALCyAFC6kBBAF/AnsBfQF/IAFBfHEhBv0MAAAAAAAAAAAAAAAAAAAAACEDQQAhAgJAA0AgAiAGTw0BIAAgAkECdGr9AAQAIQQgAyAEIAT95gH95AEhAyACQQRqIQIMAAsLIAP9HwAgA/0fAZIgA/0fAiAD/R8DkpIhBQJAA0AgAiABTw0BIAUgACACQQJ0aioCACAAIAJBAnRqKgIAlJIhBSACQQFqIQIMAAsLIAWRC8ABBQF/AnsBfQF/AX0gAkF8cSEH/QwAAAAAAAAAAAAAAAAAAAAAIQRBACEDAkADQCADIAdPDQEgACADQQJ0av0ABAAgASADQQJ0av0ABAD95QEhBSAEIAUgBf3mAf3kASEEIANBBGohAwwACwsgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEGAkADQCADIAJPDQEgACADQQJ0aioCACABIANBAnRqKgIAkyEIIAYgCCAIlJIhBiADQQFqIQMMAAsLIAYLCwAgACABIAIQApEL";
|
|
3
|
-
|
|
4
|
-
// src/vector/simd.ts
|
|
5
|
-
var wasmExports = null;
|
|
6
|
-
var f32View = null;
|
|
7
|
-
function decodeBase64(base64) {
|
|
8
|
-
if (typeof Buffer !== "undefined") {
|
|
9
|
-
return new Uint8Array(Buffer.from(base64, "base64"));
|
|
10
|
-
}
|
|
11
|
-
const binary = atob(base64);
|
|
12
|
-
const bytes = new Uint8Array(binary.length);
|
|
13
|
-
for (let i = 0; i < binary.length; i++) {
|
|
14
|
-
bytes[i] = binary.charCodeAt(i);
|
|
15
|
-
}
|
|
16
|
-
return bytes;
|
|
17
|
-
}
|
|
18
|
-
try {
|
|
19
|
-
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
20
|
-
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
21
|
-
if (typeof WebAssembly !== "undefined" && typeof WebAssembly.validate === "function" && WebAssembly.validate(wasmBuffer)) {
|
|
22
|
-
const module = new WebAssembly.Module(wasmBuffer);
|
|
23
|
-
const instance = new WebAssembly.Instance(module);
|
|
24
|
-
wasmExports = instance.exports;
|
|
25
|
-
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
26
|
-
}
|
|
27
|
-
} catch {
|
|
28
|
-
wasmExports = null;
|
|
29
|
-
f32View = null;
|
|
30
|
-
}
|
|
31
|
-
var OFFSET_B_BYTES = 16384;
|
|
32
|
-
function ensureMemory(dim) {
|
|
33
|
-
if (!wasmExports || !f32View) return;
|
|
34
|
-
const needed = OFFSET_B_BYTES + dim * 4;
|
|
35
|
-
const currentSize = wasmExports.memory.buffer.byteLength;
|
|
36
|
-
if (needed > currentSize) {
|
|
37
|
-
const pagesNeeded = Math.ceil(needed / 65536);
|
|
38
|
-
const currentPages = currentSize / 65536;
|
|
39
|
-
wasmExports.memory.grow(pagesNeeded - currentPages);
|
|
40
|
-
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
function getF32View() {
|
|
44
|
-
if (!f32View) {
|
|
45
|
-
throw new Error("SIMD memory is not initialized");
|
|
46
|
-
}
|
|
47
|
-
return f32View;
|
|
48
|
-
}
|
|
49
|
-
function copyOneVector(a) {
|
|
50
|
-
getF32View().set(a, 0);
|
|
51
|
-
}
|
|
52
|
-
function copyTwoVectors(a, b) {
|
|
53
|
-
const view = getF32View();
|
|
54
|
-
view.set(a, 0);
|
|
55
|
-
view.set(b, OFFSET_B_BYTES / 4);
|
|
56
|
-
}
|
|
57
|
-
function simdDotProduct(a, b) {
|
|
58
|
-
if (!wasmExports || !f32View) return null;
|
|
59
|
-
ensureMemory(a.length);
|
|
60
|
-
copyTwoVectors(a, b);
|
|
61
|
-
return wasmExports.dot_product(0, OFFSET_B_BYTES, a.length);
|
|
62
|
-
}
|
|
63
|
-
function simdMagnitude(a) {
|
|
64
|
-
if (!wasmExports || !f32View) return null;
|
|
65
|
-
ensureMemory(a.length);
|
|
66
|
-
copyOneVector(a);
|
|
67
|
-
return wasmExports.magnitude(0, a.length);
|
|
68
|
-
}
|
|
69
|
-
function simdEuclideanDistance(a, b) {
|
|
70
|
-
if (!wasmExports || !f32View) return null;
|
|
71
|
-
ensureMemory(a.length);
|
|
72
|
-
copyTwoVectors(a, b);
|
|
73
|
-
return wasmExports.euclidean_distance(0, OFFSET_B_BYTES, a.length);
|
|
74
|
-
}
|
|
75
|
-
function isSimdAvailable() {
|
|
76
|
-
return wasmExports !== null;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// src/core/heap.ts
|
|
80
|
-
function createMinHeap(compare) {
|
|
81
|
-
return createHeap(compare);
|
|
82
|
-
}
|
|
83
|
-
function createMaxHeap(compare) {
|
|
84
|
-
return createHeap((a, b) => compare(b, a));
|
|
85
|
-
}
|
|
86
|
-
function createBoundedMaxHeap(compare, capacity) {
|
|
87
|
-
const heap = createMaxHeap(compare);
|
|
88
|
-
const innerPush = heap.push;
|
|
89
|
-
return {
|
|
90
|
-
push(item) {
|
|
91
|
-
if (heap.size < capacity) {
|
|
92
|
-
innerPush(item);
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
const top = heap.peek();
|
|
96
|
-
if (top !== void 0 && compare(item, top) < 0) {
|
|
97
|
-
heap.pop();
|
|
98
|
-
innerPush(item);
|
|
99
|
-
}
|
|
100
|
-
},
|
|
101
|
-
pop: heap.pop,
|
|
102
|
-
peek: heap.peek,
|
|
103
|
-
get size() {
|
|
104
|
-
return heap.size;
|
|
105
|
-
},
|
|
106
|
-
toSortedArray: heap.toSortedArray
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
function createHeap(compare) {
|
|
110
|
-
const items = [];
|
|
111
|
-
function siftUp(idx) {
|
|
112
|
-
while (idx > 0) {
|
|
113
|
-
const parent = idx - 1 >> 1;
|
|
114
|
-
if (compare(items[idx], items[parent]) >= 0) break;
|
|
115
|
-
const tmp = items[idx];
|
|
116
|
-
items[idx] = items[parent];
|
|
117
|
-
items[parent] = tmp;
|
|
118
|
-
idx = parent;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
function siftDown(idx) {
|
|
122
|
-
const len = items.length;
|
|
123
|
-
while (true) {
|
|
124
|
-
let smallest = idx;
|
|
125
|
-
const left = 2 * idx + 1;
|
|
126
|
-
const right = 2 * idx + 2;
|
|
127
|
-
if (left < len && compare(items[left], items[smallest]) < 0) smallest = left;
|
|
128
|
-
if (right < len && compare(items[right], items[smallest]) < 0) smallest = right;
|
|
129
|
-
if (smallest === idx) break;
|
|
130
|
-
const tmp = items[idx];
|
|
131
|
-
items[idx] = items[smallest];
|
|
132
|
-
items[smallest] = tmp;
|
|
133
|
-
idx = smallest;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
return {
|
|
137
|
-
push(item) {
|
|
138
|
-
items.push(item);
|
|
139
|
-
siftUp(items.length - 1);
|
|
140
|
-
},
|
|
141
|
-
pop() {
|
|
142
|
-
if (items.length === 0) return void 0;
|
|
143
|
-
const top = items[0];
|
|
144
|
-
const last = items.pop();
|
|
145
|
-
if (items.length > 0) {
|
|
146
|
-
items[0] = last;
|
|
147
|
-
siftDown(0);
|
|
148
|
-
}
|
|
149
|
-
return top;
|
|
150
|
-
},
|
|
151
|
-
peek() {
|
|
152
|
-
return items[0];
|
|
153
|
-
},
|
|
154
|
-
get size() {
|
|
155
|
-
return items.length;
|
|
156
|
-
},
|
|
157
|
-
toSortedArray() {
|
|
158
|
-
return [...items].sort(compare);
|
|
159
|
-
}
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// src/vector/similarity.ts
|
|
164
|
-
function magnitude(v) {
|
|
165
|
-
const result = simdMagnitude(v);
|
|
166
|
-
if (result !== null) return result;
|
|
167
|
-
let sum = 0;
|
|
168
|
-
for (let i = 0; i < v.length; i++) {
|
|
169
|
-
sum += v[i] * v[i];
|
|
170
|
-
}
|
|
171
|
-
return Math.sqrt(sum);
|
|
172
|
-
}
|
|
173
|
-
function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
|
|
174
|
-
if (magA === 0 || magB === 0) return 0;
|
|
175
|
-
return dotProduct(a, b) / (magA * magB);
|
|
176
|
-
}
|
|
177
|
-
function dotProduct(a, b) {
|
|
178
|
-
const result = simdDotProduct(a, b);
|
|
179
|
-
if (result !== null) return result;
|
|
180
|
-
let sum = 0;
|
|
181
|
-
for (let i = 0; i < a.length; i++) {
|
|
182
|
-
sum += a[i] * b[i];
|
|
183
|
-
}
|
|
184
|
-
return sum;
|
|
185
|
-
}
|
|
186
|
-
function euclideanDistance(a, b) {
|
|
187
|
-
const result = simdEuclideanDistance(a, b);
|
|
188
|
-
if (result !== null) return result;
|
|
189
|
-
let sum = 0;
|
|
190
|
-
for (let i = 0; i < a.length; i++) {
|
|
191
|
-
const diff = a[i] - b[i];
|
|
192
|
-
sum += diff * diff;
|
|
193
|
-
}
|
|
194
|
-
return Math.sqrt(sum);
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
// src/vector/hnsw.ts
|
|
198
|
-
var MAX_LAYER_CAP = 32;
|
|
199
|
-
var COMPACTION_TOMBSTONE_RATIO = 0.1;
|
|
200
|
-
var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
|
|
201
|
-
function toDistance(a, b, magA, magB, metric) {
|
|
202
|
-
switch (metric) {
|
|
203
|
-
case "cosine":
|
|
204
|
-
return 1 - cosineSimilarityWithMagnitudes(a, b, magA, magB);
|
|
205
|
-
case "dotProduct":
|
|
206
|
-
return -dotProduct(a, b);
|
|
207
|
-
case "euclidean":
|
|
208
|
-
return euclideanDistance(a, b);
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
function toScore(distance, metric) {
|
|
212
|
-
switch (metric) {
|
|
213
|
-
case "cosine":
|
|
214
|
-
return 1 - distance;
|
|
215
|
-
case "dotProduct":
|
|
216
|
-
return -distance;
|
|
217
|
-
case "euclidean":
|
|
218
|
-
return 1 / (1 + distance);
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
var distanceAsc = (a, b) => a.distance - b.distance;
|
|
222
|
-
var distanceDesc = (a, b) => b.distance - a.distance;
|
|
223
|
-
var SQ8_OVERSELECTION_FACTOR = 2;
|
|
224
|
-
function createHNSWIndex(dimension, store, config, quantizer) {
|
|
225
|
-
const M = config?.m ?? 16;
|
|
226
|
-
const Mmax0 = M * 2;
|
|
227
|
-
const efCons = config?.efConstruction ?? 200;
|
|
228
|
-
const buildMetric = config?.metric ?? "cosine";
|
|
229
|
-
const mL = 1 / Math.log(M);
|
|
230
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
231
|
-
const tombstones = /* @__PURE__ */ new Set();
|
|
232
|
-
let entryPointId = null;
|
|
233
|
-
let topLayer = -1;
|
|
234
|
-
function randomLevel() {
|
|
235
|
-
let u = Math.random();
|
|
236
|
-
if (u === 0) u = Number.MIN_VALUE;
|
|
237
|
-
return Math.min(Math.floor(-Math.log(u) * mL), MAX_LAYER_CAP);
|
|
238
|
-
}
|
|
239
|
-
function getEntry(docId) {
|
|
240
|
-
return store.get(docId);
|
|
241
|
-
}
|
|
242
|
-
function nodeDistance(aId, bId, metric) {
|
|
243
|
-
const a = getEntry(aId);
|
|
244
|
-
const b = getEntry(bId);
|
|
245
|
-
if (!a || !b) return Number.POSITIVE_INFINITY;
|
|
246
|
-
return toDistance(a.vector, b.vector, a.magnitude, b.magnitude, metric);
|
|
247
|
-
}
|
|
248
|
-
function queryDistance(qVec, qMag, nodeId, metric) {
|
|
249
|
-
const entry = getEntry(nodeId);
|
|
250
|
-
if (!entry) return Number.POSITIVE_INFINITY;
|
|
251
|
-
return toDistance(qVec, entry.vector, qMag, entry.magnitude, metric);
|
|
252
|
-
}
|
|
253
|
-
function maxConns(layer) {
|
|
254
|
-
return layer === 0 ? Mmax0 : M;
|
|
255
|
-
}
|
|
256
|
-
function searchLayer(qVec, qMag, eps, ef, layer, metric, skipTombstones, distFn) {
|
|
257
|
-
const getDistance = distFn ?? ((nodeId) => queryDistance(qVec, qMag, nodeId, metric));
|
|
258
|
-
const visited = /* @__PURE__ */ new Set();
|
|
259
|
-
const candidates = createMinHeap(distanceAsc);
|
|
260
|
-
const results = createMinHeap(distanceDesc);
|
|
261
|
-
let furthestDist = Number.POSITIVE_INFINITY;
|
|
262
|
-
for (const epId of eps) {
|
|
263
|
-
if (visited.has(epId)) continue;
|
|
264
|
-
visited.add(epId);
|
|
265
|
-
const node = nodes.get(epId);
|
|
266
|
-
if (!node) continue;
|
|
267
|
-
if (skipTombstones && tombstones.has(epId)) continue;
|
|
268
|
-
const dist = getDistance(epId);
|
|
269
|
-
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
270
|
-
const pair = { docId: epId, distance: dist };
|
|
271
|
-
candidates.push(pair);
|
|
272
|
-
results.push(pair);
|
|
273
|
-
if (results.size > ef) {
|
|
274
|
-
results.pop();
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
const topResult = results.peek();
|
|
278
|
-
if (topResult) furthestDist = topResult.distance;
|
|
279
|
-
while (candidates.size > 0) {
|
|
280
|
-
const nearest = candidates.pop();
|
|
281
|
-
if (!nearest) break;
|
|
282
|
-
if (nearest.distance > furthestDist) break;
|
|
283
|
-
const node = nodes.get(nearest.docId);
|
|
284
|
-
if (!node || layer >= node.connections.length) continue;
|
|
285
|
-
for (const neighborId of node.connections[layer]) {
|
|
286
|
-
if (visited.has(neighborId)) continue;
|
|
287
|
-
visited.add(neighborId);
|
|
288
|
-
if (skipTombstones && tombstones.has(neighborId)) continue;
|
|
289
|
-
const neighborNode = nodes.get(neighborId);
|
|
290
|
-
if (!neighborNode) continue;
|
|
291
|
-
const dist = getDistance(neighborId);
|
|
292
|
-
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
293
|
-
if (dist < furthestDist || results.size < ef) {
|
|
294
|
-
const pair = { docId: neighborId, distance: dist };
|
|
295
|
-
candidates.push(pair);
|
|
296
|
-
results.push(pair);
|
|
297
|
-
if (results.size > ef) {
|
|
298
|
-
results.pop();
|
|
299
|
-
}
|
|
300
|
-
const newTop = results.peek();
|
|
301
|
-
if (newTop) furthestDist = newTop.distance;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
return results;
|
|
306
|
-
}
|
|
307
|
-
function nearestFromHeap(heap) {
|
|
308
|
-
let nearest;
|
|
309
|
-
while (heap.size > 0) {
|
|
310
|
-
nearest = heap.pop();
|
|
311
|
-
}
|
|
312
|
-
return nearest;
|
|
313
|
-
}
|
|
314
|
-
function selectNeighborsHeuristic(targetId, candidates, maxConnections, layer, metric, extendCandidates, keepPruned) {
|
|
315
|
-
const working = [...candidates];
|
|
316
|
-
working.sort((a, b) => a.distance - b.distance);
|
|
317
|
-
const selected = [];
|
|
318
|
-
const discarded = [];
|
|
319
|
-
for (const candidate of working) {
|
|
320
|
-
if (selected.length >= maxConnections) break;
|
|
321
|
-
let accepted = true;
|
|
322
|
-
for (const sel of selected) {
|
|
323
|
-
const distBetween = nodeDistance(candidate.docId, sel.docId, metric);
|
|
324
|
-
if (candidate.distance >= distBetween) {
|
|
325
|
-
accepted = false;
|
|
326
|
-
break;
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
if (accepted) {
|
|
330
|
-
selected.push(candidate);
|
|
331
|
-
} else {
|
|
332
|
-
discarded.push(candidate);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
{
|
|
336
|
-
for (const disc of discarded) {
|
|
337
|
-
if (selected.length >= maxConnections) break;
|
|
338
|
-
selected.push(disc);
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
return selected;
|
|
342
|
-
}
|
|
343
|
-
function pruneConnections(nodeId, node, layer, metric) {
|
|
344
|
-
const mc = maxConns(layer);
|
|
345
|
-
if (node.connections[layer].size <= mc) return;
|
|
346
|
-
const conns = [];
|
|
347
|
-
for (const connId of node.connections[layer]) {
|
|
348
|
-
const dist = nodeDistance(nodeId, connId, metric);
|
|
349
|
-
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
350
|
-
conns.push({ docId: connId, distance: dist });
|
|
351
|
-
}
|
|
352
|
-
const pruned = selectNeighborsHeuristic(nodeId, conns, mc, layer, metric);
|
|
353
|
-
node.connections[layer] = new Set(pruned.map((p) => p.docId));
|
|
354
|
-
}
|
|
355
|
-
function insertNode(docId) {
|
|
356
|
-
const vecEntry = store.get(docId);
|
|
357
|
-
if (!vecEntry) {
|
|
358
|
-
throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
|
|
359
|
-
}
|
|
360
|
-
if (vecEntry.vector.length !== dimension) {
|
|
361
|
-
throw new Error(`Vector dimension mismatch: expected ${dimension}, got ${vecEntry.vector.length}`);
|
|
362
|
-
}
|
|
363
|
-
if (nodes.has(docId)) {
|
|
364
|
-
removeNodeEager(docId);
|
|
365
|
-
}
|
|
366
|
-
tombstones.delete(docId);
|
|
367
|
-
const l = randomLevel();
|
|
368
|
-
const node = {
|
|
369
|
-
docId,
|
|
370
|
-
maxLayer: l,
|
|
371
|
-
connections: Array.from({ length: l + 1 }, () => /* @__PURE__ */ new Set())
|
|
372
|
-
};
|
|
373
|
-
nodes.set(docId, node);
|
|
374
|
-
if (entryPointId === null) {
|
|
375
|
-
entryPointId = docId;
|
|
376
|
-
topLayer = l;
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
const metric = buildMetric;
|
|
380
|
-
let currentEPs = [entryPointId];
|
|
381
|
-
for (let layer = topLayer; layer > l; layer--) {
|
|
382
|
-
const heap = searchLayer(vecEntry.vector, vecEntry.magnitude, currentEPs, 1, layer, metric, false);
|
|
383
|
-
const nearest = nearestFromHeap(heap);
|
|
384
|
-
if (nearest) {
|
|
385
|
-
currentEPs = [nearest.docId];
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
for (let layer = Math.min(l, topLayer); layer >= 0; layer--) {
|
|
389
|
-
const heap = searchLayer(vecEntry.vector, vecEntry.magnitude, currentEPs, efCons, layer, metric, false);
|
|
390
|
-
const candidates = heap.toSortedArray().reverse();
|
|
391
|
-
const mc = maxConns(layer);
|
|
392
|
-
const neighbors = selectNeighborsHeuristic(docId, candidates, mc, layer, metric);
|
|
393
|
-
for (const neighbor of neighbors) {
|
|
394
|
-
node.connections[layer].add(neighbor.docId);
|
|
395
|
-
const neighborNode = nodes.get(neighbor.docId);
|
|
396
|
-
if (neighborNode && layer < neighborNode.connections.length) {
|
|
397
|
-
neighborNode.connections[layer].add(docId);
|
|
398
|
-
pruneConnections(neighbor.docId, neighborNode, layer, metric);
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
if (candidates.length > 0) {
|
|
402
|
-
currentEPs = candidates.map((n) => n.docId);
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
if (l > topLayer) {
|
|
406
|
-
entryPointId = docId;
|
|
407
|
-
topLayer = l;
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
function removeNodeEager(docId, excludeDocIds) {
|
|
411
|
-
const node = nodes.get(docId);
|
|
412
|
-
if (!node) return;
|
|
413
|
-
const metric = buildMetric;
|
|
414
|
-
for (let layer = 0; layer <= node.maxLayer; layer++) {
|
|
415
|
-
if (layer >= node.connections.length) continue;
|
|
416
|
-
const formerNeighborIds = new Set(node.connections[layer]);
|
|
417
|
-
for (const neighborId of formerNeighborIds) {
|
|
418
|
-
const neighborNode = nodes.get(neighborId);
|
|
419
|
-
if (neighborNode && layer < neighborNode.connections.length) {
|
|
420
|
-
neighborNode.connections[layer].delete(docId);
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
for (const neighborId of formerNeighborIds) {
|
|
424
|
-
const neighborNode = nodes.get(neighborId);
|
|
425
|
-
if (!neighborNode || layer >= neighborNode.connections.length) continue;
|
|
426
|
-
const mc = maxConns(layer);
|
|
427
|
-
if (neighborNode.connections[layer].size >= mc) continue;
|
|
428
|
-
const candidateIds = new Set(neighborNode.connections[layer]);
|
|
429
|
-
for (const otherId of formerNeighborIds) {
|
|
430
|
-
if (otherId !== neighborId && otherId !== docId) {
|
|
431
|
-
if (excludeDocIds?.has(otherId)) continue;
|
|
432
|
-
candidateIds.add(otherId);
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
const candidates = [];
|
|
436
|
-
for (const candId of candidateIds) {
|
|
437
|
-
const dist = nodeDistance(neighborId, candId, metric);
|
|
438
|
-
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
439
|
-
candidates.push({ docId: candId, distance: dist });
|
|
440
|
-
}
|
|
441
|
-
const selected = selectNeighborsHeuristic(neighborId, candidates, mc, layer, metric);
|
|
442
|
-
const newConns = new Set(selected.map((s) => s.docId));
|
|
443
|
-
neighborNode.connections[layer] = newConns;
|
|
444
|
-
for (const newConnId of newConns) {
|
|
445
|
-
const newConnNode = nodes.get(newConnId);
|
|
446
|
-
if (newConnNode && layer < newConnNode.connections.length) {
|
|
447
|
-
newConnNode.connections[layer].add(neighborId);
|
|
448
|
-
pruneConnections(newConnId, newConnNode, layer, metric);
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
nodes.delete(docId);
|
|
454
|
-
if (entryPointId === docId) {
|
|
455
|
-
if (nodes.size === 0) {
|
|
456
|
-
entryPointId = null;
|
|
457
|
-
topLayer = -1;
|
|
458
|
-
return;
|
|
459
|
-
}
|
|
460
|
-
let newEntryId = null;
|
|
461
|
-
let newTopLayer = -1;
|
|
462
|
-
for (const [nId, n] of nodes) {
|
|
463
|
-
if (tombstones.has(nId)) continue;
|
|
464
|
-
if (n.maxLayer > newTopLayer) {
|
|
465
|
-
newTopLayer = n.maxLayer;
|
|
466
|
-
newEntryId = nId;
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
if (newEntryId === null) {
|
|
470
|
-
for (const [nId, n] of nodes) {
|
|
471
|
-
if (n.maxLayer > newTopLayer) {
|
|
472
|
-
newTopLayer = n.maxLayer;
|
|
473
|
-
newEntryId = nId;
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
entryPointId = newEntryId;
|
|
478
|
-
topLayer = newTopLayer;
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
function markTombstone(docId) {
|
|
482
|
-
if (!nodes.has(docId)) return;
|
|
483
|
-
tombstones.add(docId);
|
|
484
|
-
if (entryPointId === docId) {
|
|
485
|
-
let newEntryId = null;
|
|
486
|
-
let newTopLayer = -1;
|
|
487
|
-
for (const [nId, n] of nodes) {
|
|
488
|
-
if (tombstones.has(nId)) continue;
|
|
489
|
-
if (n.maxLayer > newTopLayer) {
|
|
490
|
-
newTopLayer = n.maxLayer;
|
|
491
|
-
newEntryId = nId;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
if (newEntryId !== null) {
|
|
495
|
-
entryPointId = newEntryId;
|
|
496
|
-
topLayer = newTopLayer;
|
|
497
|
-
} else {
|
|
498
|
-
entryPointId = null;
|
|
499
|
-
topLayer = -1;
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
function search(query, k, searchMetric, minSimilarity, filterDocIds, efSearch) {
|
|
504
|
-
if (query.length !== dimension) {
|
|
505
|
-
throw new Error(`Query dimension mismatch: expected ${dimension}, got ${query.length}`);
|
|
506
|
-
}
|
|
507
|
-
const liveSize = nodes.size - tombstones.size;
|
|
508
|
-
if (entryPointId === null || liveSize === 0) {
|
|
509
|
-
return [];
|
|
510
|
-
}
|
|
511
|
-
const useQuantized = quantizer?.isCalibrated() === true && quantizer.size > 0;
|
|
512
|
-
const defaultEf = 50;
|
|
513
|
-
let ef = Math.max(efSearch ?? defaultEf, k);
|
|
514
|
-
if (filterDocIds && filterDocIds.size < liveSize) {
|
|
515
|
-
const selectivity = filterDocIds.size / liveSize;
|
|
516
|
-
ef = Math.max(ef, Math.ceil(k / Math.max(selectivity, 0.01)));
|
|
517
|
-
ef = Math.min(ef, liveSize);
|
|
518
|
-
}
|
|
519
|
-
const qMag = magnitude(query);
|
|
520
|
-
let quantizedDistFn;
|
|
521
|
-
let prepared = null;
|
|
522
|
-
if (useQuantized && quantizer) {
|
|
523
|
-
prepared = quantizer.prepareQuery(query);
|
|
524
|
-
if (prepared) {
|
|
525
|
-
const p = prepared;
|
|
526
|
-
const metric = searchMetric;
|
|
527
|
-
const q = quantizer;
|
|
528
|
-
quantizedDistFn = (nodeId) => q.distanceFromPrepared(p, nodeId, metric);
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
let currentEPs = [entryPointId];
|
|
532
|
-
for (let layer = topLayer; layer >= 1; layer--) {
|
|
533
|
-
const heap = searchLayer(query, qMag, currentEPs, 1, layer, searchMetric, true, quantizedDistFn);
|
|
534
|
-
const nearest = nearestFromHeap(heap);
|
|
535
|
-
if (nearest) {
|
|
536
|
-
currentEPs = [nearest.docId];
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
const candidateHeap = searchLayer(query, qMag, currentEPs, ef, 0, searchMetric, true, quantizedDistFn);
|
|
540
|
-
const candidateArray = candidateHeap.toSortedArray().reverse();
|
|
541
|
-
if (useQuantized) {
|
|
542
|
-
return rerankWithFullPrecision(candidateArray, query, qMag, k, searchMetric, minSimilarity, filterDocIds);
|
|
543
|
-
}
|
|
544
|
-
const results = [];
|
|
545
|
-
for (const cand of candidateArray) {
|
|
546
|
-
if (filterDocIds && !filterDocIds.has(cand.docId)) continue;
|
|
547
|
-
const score = toScore(cand.distance, searchMetric);
|
|
548
|
-
if (score < minSimilarity) continue;
|
|
549
|
-
results.push({
|
|
550
|
-
docId: cand.docId,
|
|
551
|
-
score,
|
|
552
|
-
termFrequencies: {},
|
|
553
|
-
fieldLengths: {},
|
|
554
|
-
idf: {}
|
|
555
|
-
});
|
|
556
|
-
}
|
|
557
|
-
results.sort((a, b) => b.score - a.score || a.docId.localeCompare(b.docId));
|
|
558
|
-
return results.slice(0, k);
|
|
559
|
-
}
|
|
560
|
-
function rerankWithFullPrecision(candidates, query, qMag, k, metric, minSimilarity, filterDocIds) {
|
|
561
|
-
const reranked = [];
|
|
562
|
-
const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR, 10);
|
|
563
|
-
for (const cand of candidates) {
|
|
564
|
-
if (filterDocIds && !filterDocIds.has(cand.docId)) continue;
|
|
565
|
-
const entry = getEntry(cand.docId);
|
|
566
|
-
if (!entry) continue;
|
|
567
|
-
const fullDistance = toDistance(query, entry.vector, qMag, entry.magnitude, metric);
|
|
568
|
-
const score = toScore(fullDistance, metric);
|
|
569
|
-
if (score < minSimilarity) continue;
|
|
570
|
-
reranked.push({
|
|
571
|
-
docId: cand.docId,
|
|
572
|
-
score,
|
|
573
|
-
termFrequencies: {},
|
|
574
|
-
fieldLengths: {},
|
|
575
|
-
idf: {}
|
|
576
|
-
});
|
|
577
|
-
if (reranked.length >= rerankLimit) break;
|
|
578
|
-
}
|
|
579
|
-
reranked.sort((a, b) => b.score - a.score || a.docId.localeCompare(b.docId));
|
|
580
|
-
return reranked.slice(0, k);
|
|
581
|
-
}
|
|
582
|
-
function serialize() {
|
|
583
|
-
const nodeArray = [];
|
|
584
|
-
for (const [docId, node] of nodes) {
|
|
585
|
-
if (tombstones.has(docId)) continue;
|
|
586
|
-
const layerConns = [];
|
|
587
|
-
for (let layer = 0; layer < node.connections.length; layer++) {
|
|
588
|
-
const liveNeighbors = Array.from(node.connections[layer]).filter((id) => !tombstones.has(id));
|
|
589
|
-
if (liveNeighbors.length > 0) {
|
|
590
|
-
layerConns.push([layer, liveNeighbors]);
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
nodeArray.push([docId, node.maxLayer, layerConns]);
|
|
594
|
-
}
|
|
595
|
-
return {
|
|
596
|
-
entryPoint: entryPointId,
|
|
597
|
-
maxLayer: topLayer,
|
|
598
|
-
m: M,
|
|
599
|
-
efConstruction: efCons,
|
|
600
|
-
metric: buildMetric,
|
|
601
|
-
nodes: nodeArray
|
|
602
|
-
};
|
|
603
|
-
}
|
|
604
|
-
function deserialize(data) {
|
|
605
|
-
nodes.clear();
|
|
606
|
-
tombstones.clear();
|
|
607
|
-
for (const [docId, maxLayer, layerConns] of data.nodes) {
|
|
608
|
-
const vecData = store.get(docId);
|
|
609
|
-
if (!vecData) continue;
|
|
610
|
-
const connections = Array.from({ length: maxLayer + 1 }, () => /* @__PURE__ */ new Set());
|
|
611
|
-
for (const [layer, neighbors] of layerConns) {
|
|
612
|
-
if (layer < connections.length) {
|
|
613
|
-
connections[layer] = new Set(neighbors);
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
nodes.set(docId, { docId, maxLayer, connections });
|
|
617
|
-
}
|
|
618
|
-
if (data.entryPoint != null && data.entryPoint !== "" && nodes.has(data.entryPoint)) {
|
|
619
|
-
entryPointId = data.entryPoint;
|
|
620
|
-
topLayer = data.maxLayer;
|
|
621
|
-
} else if (nodes.size > 0) {
|
|
622
|
-
let bestId = null;
|
|
623
|
-
let bestLayer = -1;
|
|
624
|
-
for (const [nId, n] of nodes) {
|
|
625
|
-
if (n.maxLayer > bestLayer) {
|
|
626
|
-
bestLayer = n.maxLayer;
|
|
627
|
-
bestId = nId;
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
entryPointId = bestId;
|
|
631
|
-
topLayer = bestLayer;
|
|
632
|
-
} else {
|
|
633
|
-
entryPointId = null;
|
|
634
|
-
topLayer = -1;
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
function clear() {
|
|
638
|
-
nodes.clear();
|
|
639
|
-
tombstones.clear();
|
|
640
|
-
entryPointId = null;
|
|
641
|
-
topLayer = -1;
|
|
642
|
-
}
|
|
643
|
-
function* entriesIterator() {
|
|
644
|
-
for (const [docId] of nodes) {
|
|
645
|
-
if (tombstones.has(docId)) continue;
|
|
646
|
-
const entry = store.get(docId);
|
|
647
|
-
if (!entry) continue;
|
|
648
|
-
yield [docId, { docId, vector: entry.vector, magnitude: entry.magnitude }];
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
function compactionNeeded() {
|
|
652
|
-
if (nodes.size === 0) return false;
|
|
653
|
-
return tombstones.size / nodes.size > COMPACTION_TOMBSTONE_RATIO || tombstones.size > COMPACTION_ABSOLUTE_THRESHOLD;
|
|
654
|
-
}
|
|
655
|
-
function compactTombstones() {
|
|
656
|
-
if (tombstones.size === 0) return;
|
|
657
|
-
const tombstonedDocIds = Array.from(tombstones);
|
|
658
|
-
for (const docId of tombstonedDocIds) {
|
|
659
|
-
removeNodeEager(docId, tombstones);
|
|
660
|
-
}
|
|
661
|
-
tombstones.clear();
|
|
662
|
-
}
|
|
663
|
-
function rebuild() {
|
|
664
|
-
if (tombstones.size === 0 && nodes.size === 0) return;
|
|
665
|
-
const liveDocIds = [];
|
|
666
|
-
for (const [docId] of nodes) {
|
|
667
|
-
if (!tombstones.has(docId)) {
|
|
668
|
-
liveDocIds.push(docId);
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
clear();
|
|
672
|
-
for (const docId of liveDocIds) {
|
|
673
|
-
const entry = store.get(docId);
|
|
674
|
-
if (!entry) continue;
|
|
675
|
-
insertNode(docId);
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
return {
|
|
679
|
-
get dimension() {
|
|
680
|
-
return dimension;
|
|
681
|
-
},
|
|
682
|
-
get size() {
|
|
683
|
-
return nodes.size - tombstones.size;
|
|
684
|
-
},
|
|
685
|
-
get tombstoneCount() {
|
|
686
|
-
return tombstones.size;
|
|
687
|
-
},
|
|
688
|
-
get entryPointId() {
|
|
689
|
-
return entryPointId;
|
|
690
|
-
},
|
|
691
|
-
get topLayer() {
|
|
692
|
-
return topLayer;
|
|
693
|
-
},
|
|
694
|
-
get m() {
|
|
695
|
-
return M;
|
|
696
|
-
},
|
|
697
|
-
get efConstruction() {
|
|
698
|
-
return efCons;
|
|
699
|
-
},
|
|
700
|
-
get metric() {
|
|
701
|
-
return buildMetric;
|
|
702
|
-
},
|
|
703
|
-
insertNode,
|
|
704
|
-
markTombstone,
|
|
705
|
-
has: (docId) => nodes.has(docId) && !tombstones.has(docId),
|
|
706
|
-
isTombstoned: (docId) => tombstones.has(docId),
|
|
707
|
-
search,
|
|
708
|
-
clear,
|
|
709
|
-
entries: entriesIterator,
|
|
710
|
-
compactionNeeded,
|
|
711
|
-
compactTombstones,
|
|
712
|
-
rebuild,
|
|
713
|
-
serialize,
|
|
714
|
-
deserialize
|
|
715
|
-
};
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
// src/vector/vector-store.ts
|
|
719
|
-
function createVectorStore() {
|
|
720
|
-
const vectors = /* @__PURE__ */ new Map();
|
|
721
|
-
return {
|
|
722
|
-
get size() {
|
|
723
|
-
return vectors.size;
|
|
724
|
-
},
|
|
725
|
-
insert(docId, vector) {
|
|
726
|
-
const mag = magnitude(vector);
|
|
727
|
-
vectors.set(docId, { vector, magnitude: mag });
|
|
728
|
-
},
|
|
729
|
-
remove(docId) {
|
|
730
|
-
vectors.delete(docId);
|
|
731
|
-
},
|
|
732
|
-
get(docId) {
|
|
733
|
-
return vectors.get(docId);
|
|
734
|
-
},
|
|
735
|
-
has(docId) {
|
|
736
|
-
return vectors.has(docId);
|
|
737
|
-
},
|
|
738
|
-
entries() {
|
|
739
|
-
return vectors.entries();
|
|
740
|
-
},
|
|
741
|
-
clear() {
|
|
742
|
-
vectors.clear();
|
|
743
|
-
},
|
|
744
|
-
estimateMemory(dimension) {
|
|
745
|
-
const count = vectors.size;
|
|
746
|
-
if (count === 0) return 0;
|
|
747
|
-
const MAP_OVERHEAD = 64;
|
|
748
|
-
const MAP_ENTRY = 72;
|
|
749
|
-
const ENTRY_OBJ = 32;
|
|
750
|
-
const AVG_DOCID_BYTES = 56;
|
|
751
|
-
const TYPED_ARRAY_HEADER = 64;
|
|
752
|
-
const MAGNITUDE_BYTES = 8;
|
|
753
|
-
const perEntry = MAP_ENTRY + ENTRY_OBJ + AVG_DOCID_BYTES + TYPED_ARRAY_HEADER + MAGNITUDE_BYTES;
|
|
754
|
-
return MAP_OVERHEAD + count * (perEntry + dimension * 4);
|
|
755
|
-
}
|
|
756
|
-
};
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
export { cosineSimilarityWithMagnitudes, createBoundedMaxHeap, createHNSWIndex, createVectorStore, dotProduct, euclideanDistance, isSimdAvailable, magnitude };
|