@delali/narsil 0.3.0 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -0,0 +1,4358 @@
|
|
|
1
|
+
import { unpackLevels, osqRecordBytes, VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes, writeRecord } from './chunk-RIJSLY2I.mjs';
|
|
2
|
+
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_FILE_BYTES, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, PLACEMENT_ROOM_REQUESTS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, OSQ_PLACEMENT_QUERY_BITS, GRAPH_EXCLUSIVE_WAIT_DEADLINE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, compareCodePoints, INITIAL_UPPER_SLOTS, GRAPH_LOCK_WAIT_SLICE_MS, threadSlotGone, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, DEFAULT_EF_SEARCH, OSQ_LAMBDA, FARTHEST_REJECTED_LINKS_KEPT, MAX_DOC_ID_TABLE_BYTES } from './chunk-BDOSMEEP.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes, MAX_DOC_ID_LENGTH } from './chunk-RHG4O77A.mjs';
|
|
4
|
+
|
|
5
|
+
// src/vector/shared-buffers/growable.ts
|
|
6
|
+
function sharedMemoryAvailable() {
|
|
7
|
+
return typeof SharedArrayBuffer === "function";
|
|
8
|
+
}
|
|
9
|
+
function createGrowableBuffer(initialBytes, maxBytes) {
|
|
10
|
+
const initial = Math.min(initialBytes, maxBytes);
|
|
11
|
+
if (sharedMemoryAvailable()) {
|
|
12
|
+
return new SharedArrayBuffer(initial, { maxByteLength: maxBytes });
|
|
13
|
+
}
|
|
14
|
+
return new ArrayBuffer(initial, { maxByteLength: maxBytes });
|
|
15
|
+
}
|
|
16
|
+
function createFixedBuffer(bytes) {
|
|
17
|
+
return sharedMemoryAvailable() ? new SharedArrayBuffer(bytes) : new ArrayBuffer(bytes);
|
|
18
|
+
}
|
|
19
|
+
function nextGrowthTarget(have, needed, max) {
|
|
20
|
+
return Math.min(max, Math.max(needed, have * 2));
|
|
21
|
+
}
|
|
22
|
+
function growBufferTo(buffer, bytes) {
|
|
23
|
+
const held = buffer.byteLength;
|
|
24
|
+
if (held >= bytes) return;
|
|
25
|
+
resizeBuffer(buffer, bytes, nextGrowthTarget(held, bytes, buffer.maxByteLength));
|
|
26
|
+
}
|
|
27
|
+
function growBufferToExactly(buffer, bytes) {
|
|
28
|
+
if (buffer.byteLength >= bytes) return;
|
|
29
|
+
resizeBuffer(buffer, bytes, bytes);
|
|
30
|
+
}
|
|
31
|
+
function resizeBuffer(buffer, bytes, target) {
|
|
32
|
+
if (bytes > buffer.maxByteLength) {
|
|
33
|
+
throw new NarsilError(
|
|
34
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
35
|
+
`A vector structure needs ${bytes} bytes where it may hold at most ${buffer.maxByteLength}`,
|
|
36
|
+
{ bytes, maxBytes: buffer.maxByteLength }
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
if (buffer instanceof ArrayBuffer) {
|
|
40
|
+
buffer.resize(target);
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
buffer.grow(target);
|
|
45
|
+
} catch (err) {
|
|
46
|
+
if (buffer.byteLength < bytes) throw err;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
function fixedView(buffer, kind) {
|
|
50
|
+
return new kind(buffer, 0, Math.floor(buffer.byteLength / kind.BYTES_PER_ELEMENT));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// src/vector/hnsw/handles.ts
|
|
54
|
+
var GRAPH_ENTRY_POINT = 0;
|
|
55
|
+
var GRAPH_TOP_LAYER = 1;
|
|
56
|
+
var GRAPH_M = 2;
|
|
57
|
+
var GRAPH_MMAX0 = 3;
|
|
58
|
+
var GRAPH_EF_CONSTRUCTION = 4;
|
|
59
|
+
var GRAPH_METRIC = 5;
|
|
60
|
+
var GRAPH_RETIRED = 6;
|
|
61
|
+
var GRAPH_NODE_COUNT = 32;
|
|
62
|
+
var GRAPH_TOMBSTONE_COUNT = 33;
|
|
63
|
+
var GRAPH_UNLINKED_NODES = 34;
|
|
64
|
+
var GRAPH_INCOMING_UNCOUNTED = 35;
|
|
65
|
+
var GRAPH_UPPER_USED = 64;
|
|
66
|
+
var GRAPH_SLOTS = 65;
|
|
67
|
+
var GRAPH_UPPER_CAPACITY = 66;
|
|
68
|
+
var GRAPH_SLOT_CAPACITY = 67;
|
|
69
|
+
var GRAPH_LOCK = 96;
|
|
70
|
+
var GRAPH_WRITERS_WAITING = 97;
|
|
71
|
+
var GRAPH_ENTRY_LOCK = 128;
|
|
72
|
+
var GRAPH_HEADER_WORDS = 160;
|
|
73
|
+
var HELD_WRITE = 0;
|
|
74
|
+
var HELD_WRITE_VERSION = 1;
|
|
75
|
+
var HELD_GRAPH = 2;
|
|
76
|
+
var HELD_FENCE = 3;
|
|
77
|
+
var HELD_GRAPH_WAITING = 4;
|
|
78
|
+
var HELD_WORDS_PER_THREAD = 32;
|
|
79
|
+
var METRIC_CODES = ["cosine", "dotProduct", "euclidean"];
|
|
80
|
+
function perOrdinal(bytesPerOrdinal, initialOrdinals) {
|
|
81
|
+
return createGrowableBuffer(initialOrdinals * bytesPerOrdinal, MAX_VECTOR_ORDINALS * bytesPerOrdinal);
|
|
82
|
+
}
|
|
83
|
+
function createSharedGraphHandles(shape) {
|
|
84
|
+
const header = new Int32Array(createFixedBuffer(GRAPH_HEADER_WORDS * 4), 0, GRAPH_HEADER_WORDS);
|
|
85
|
+
header[GRAPH_ENTRY_POINT] = -1;
|
|
86
|
+
header[GRAPH_TOP_LAYER] = -1;
|
|
87
|
+
header[GRAPH_M] = shape.m;
|
|
88
|
+
header[GRAPH_MMAX0] = shape.mMax0;
|
|
89
|
+
header[GRAPH_EF_CONSTRUCTION] = shape.efConstruction;
|
|
90
|
+
header[GRAPH_METRIC] = Math.max(0, METRIC_CODES.indexOf(shape.metric));
|
|
91
|
+
const level0Stride = shape.mMax0 + 2;
|
|
92
|
+
const upperStride = shape.m + 2;
|
|
93
|
+
const heldWords = VECTOR_SCRATCH_SLOTS * HELD_WORDS_PER_THREAD;
|
|
94
|
+
const handles = {
|
|
95
|
+
header,
|
|
96
|
+
nodeLevels: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
97
|
+
level0: perOrdinal(level0Stride * 4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
98
|
+
upperBase: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
99
|
+
upper: perOrdinal(upperStride * 4, INITIAL_UPPER_SLOTS),
|
|
100
|
+
locks: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
101
|
+
tombstones: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
102
|
+
incoming: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
103
|
+
heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
|
|
104
|
+
};
|
|
105
|
+
publishGraphCapacity(handles);
|
|
106
|
+
return handles;
|
|
107
|
+
}
|
|
108
|
+
function raiseHeaderWord(header, word, value) {
|
|
109
|
+
for (; ; ) {
|
|
110
|
+
const seen = Atomics.load(header, word);
|
|
111
|
+
if (seen >= value) return;
|
|
112
|
+
if (Atomics.compareExchange(header, word, seen, value) === seen) return;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function publishGraphCapacity(handles) {
|
|
116
|
+
const level0Stride = handles.header[GRAPH_MMAX0] + 2;
|
|
117
|
+
const slotCapacity = Math.min(
|
|
118
|
+
handles.nodeLevels.byteLength,
|
|
119
|
+
Math.floor(handles.level0.byteLength / (level0Stride * 4)),
|
|
120
|
+
Math.floor(handles.upperBase.byteLength / 4),
|
|
121
|
+
Math.floor(handles.locks.byteLength / 4),
|
|
122
|
+
handles.tombstones.byteLength,
|
|
123
|
+
Math.floor(handles.incoming.byteLength / 4)
|
|
124
|
+
);
|
|
125
|
+
raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
|
|
126
|
+
raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
|
|
127
|
+
}
|
|
128
|
+
function graphShapeOf(header) {
|
|
129
|
+
return {
|
|
130
|
+
m: header[GRAPH_M],
|
|
131
|
+
mMax0: header[GRAPH_MMAX0],
|
|
132
|
+
efConstruction: header[GRAPH_EF_CONSTRUCTION],
|
|
133
|
+
metric: METRIC_CODES[header[GRAPH_METRIC]] ?? "cosine"
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// src/vector/hnsw/locks.ts
|
|
138
|
+
var FREE = 0;
|
|
139
|
+
var EXCLUSIVE = -1;
|
|
140
|
+
function openGraphLocks(handles, threadSlot) {
|
|
141
|
+
let words = fixedView(handles.locks, Int32Array);
|
|
142
|
+
return {
|
|
143
|
+
threadSlot,
|
|
144
|
+
header: handles.header,
|
|
145
|
+
held: handles.heldLocks,
|
|
146
|
+
words(ord) {
|
|
147
|
+
if (ord < words.length) return words;
|
|
148
|
+
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
149
|
+
words = fixedView(handles.locks, Int32Array);
|
|
150
|
+
return words;
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
var blockingAllowed = typeof document === "undefined";
|
|
155
|
+
function canWait(words) {
|
|
156
|
+
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
157
|
+
}
|
|
158
|
+
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
159
|
+
if (!canWait(words)) return;
|
|
160
|
+
Atomics.wait(words, index, seen, sliceMs);
|
|
161
|
+
}
|
|
162
|
+
function heldIndex(locks, kind) {
|
|
163
|
+
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
164
|
+
}
|
|
165
|
+
function acquireExclusive(words, index) {
|
|
166
|
+
let spins = 0;
|
|
167
|
+
for (; ; ) {
|
|
168
|
+
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
169
|
+
if (seen === FREE) return;
|
|
170
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
171
|
+
spins += 1;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
function acquireExclusiveWithin(words, index, deadlineMs) {
|
|
178
|
+
let spins = 0;
|
|
179
|
+
let deadline = Number.NaN;
|
|
180
|
+
for (; ; ) {
|
|
181
|
+
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
182
|
+
if (seen === FREE) return true;
|
|
183
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
184
|
+
spins += 1;
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const now = performance.now();
|
|
188
|
+
if (Number.isNaN(deadline)) deadline = now + deadlineMs;
|
|
189
|
+
else if (now >= deadline) return false;
|
|
190
|
+
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
function releaseExclusive(words, index) {
|
|
194
|
+
Atomics.store(words, index, FREE);
|
|
195
|
+
if (canWait(words)) Atomics.notify(words, index);
|
|
196
|
+
}
|
|
197
|
+
var WRITE_HELD = 1;
|
|
198
|
+
var WRITER_WAITING = 2;
|
|
199
|
+
var VERSION_STEP = 4;
|
|
200
|
+
function beginNodeRead(locks, ord) {
|
|
201
|
+
const words = locks.words(ord);
|
|
202
|
+
for (; ; ) {
|
|
203
|
+
const seen = Atomics.load(words, ord);
|
|
204
|
+
if ((seen & WRITE_HELD) === 0) return seen;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
function nodeReadHeld(locks, ord, version) {
|
|
208
|
+
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
209
|
+
return Atomics.load(locks.words(ord), ord) === version;
|
|
210
|
+
}
|
|
211
|
+
function lockNodeWrite(locks, ord) {
|
|
212
|
+
const words = locks.words(ord);
|
|
213
|
+
let spins = 0;
|
|
214
|
+
for (; ; ) {
|
|
215
|
+
const seen = Atomics.load(words, ord);
|
|
216
|
+
if ((seen & WRITE_HELD) === 0) {
|
|
217
|
+
const claimed = seen | WRITE_HELD;
|
|
218
|
+
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
219
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
220
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
226
|
+
spins += 1;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const waiting = seen | WRITER_WAITING;
|
|
230
|
+
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
231
|
+
waitOn(words, ord, waiting);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
function releaseWrite(words, ord) {
|
|
236
|
+
const seen = Atomics.load(words, ord);
|
|
237
|
+
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
238
|
+
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
239
|
+
}
|
|
240
|
+
function unlockNodeWrite(locks, ord) {
|
|
241
|
+
releaseWrite(locks.words(ord), ord);
|
|
242
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
243
|
+
}
|
|
244
|
+
function lockGraphShared(locks) {
|
|
245
|
+
const { header } = locks;
|
|
246
|
+
let spins = 0;
|
|
247
|
+
for (; ; ) {
|
|
248
|
+
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
249
|
+
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
250
|
+
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
251
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
257
|
+
spins += 1;
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
function unlockGraphShared(locks) {
|
|
264
|
+
const { header } = locks;
|
|
265
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
266
|
+
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
267
|
+
}
|
|
268
|
+
function lockGraphExclusive(locks) {
|
|
269
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
270
|
+
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
271
|
+
try {
|
|
272
|
+
if (acquireExclusiveWithin(locks.header, GRAPH_LOCK, GRAPH_EXCLUSIVE_WAIT_DEADLINE_MS)) return;
|
|
273
|
+
const released = releaseGoneThreads(locks);
|
|
274
|
+
if (released > 0 && acquireExclusiveWithin(locks.header, GRAPH_LOCK, GRAPH_EXCLUSIVE_WAIT_DEADLINE_MS)) return;
|
|
275
|
+
throw new NarsilError(
|
|
276
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
277
|
+
`Another thread held the vector graph for ${GRAPH_EXCLUSIVE_WAIT_DEADLINE_MS} ms, so this change to the graph gave up`,
|
|
278
|
+
{ waitedMs: GRAPH_EXCLUSIVE_WAIT_DEADLINE_MS, threadSlot: locks.threadSlot }
|
|
279
|
+
);
|
|
280
|
+
} finally {
|
|
281
|
+
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
282
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
function unlockGraphExclusive(locks) {
|
|
286
|
+
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
287
|
+
}
|
|
288
|
+
function lockEntry(locks) {
|
|
289
|
+
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
290
|
+
}
|
|
291
|
+
function unlockEntry(locks) {
|
|
292
|
+
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
293
|
+
}
|
|
294
|
+
function releaseGoneThreads(locks) {
|
|
295
|
+
let released = 0;
|
|
296
|
+
const slots = locks.held.length / HELD_WORDS_PER_THREAD;
|
|
297
|
+
for (let slot = 0; slot < slots; slot++) {
|
|
298
|
+
if (slot === locks.threadSlot || !threadSlotGone(slot)) continue;
|
|
299
|
+
const base = slot * HELD_WORDS_PER_THREAD;
|
|
300
|
+
const writeOrd = Atomics.load(locks.held, base + HELD_WRITE) - 1;
|
|
301
|
+
const holdsGraph = Atomics.load(locks.held, base + HELD_GRAPH) === 1;
|
|
302
|
+
const waitsForGraph = Atomics.load(locks.held, base + HELD_GRAPH_WAITING) === 1;
|
|
303
|
+
if (!holdsGraph && !waitsForGraph && writeOrd < 0) continue;
|
|
304
|
+
releaseSlot(locks.header, locks.held, writeOrd >= 0 ? locks.words(writeOrd) : null, slot);
|
|
305
|
+
released += 1;
|
|
306
|
+
}
|
|
307
|
+
return released;
|
|
308
|
+
}
|
|
309
|
+
function releaseLocksHeldBy(handles, threadSlot) {
|
|
310
|
+
releaseSlot(handles.header, handles.heldLocks, new Int32Array(handles.locks), threadSlot);
|
|
311
|
+
}
|
|
312
|
+
function releaseSlot(header, held, words, threadSlot) {
|
|
313
|
+
const base = threadSlot * HELD_WORDS_PER_THREAD;
|
|
314
|
+
if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
|
|
315
|
+
Atomics.sub(header, GRAPH_WRITERS_WAITING, 1);
|
|
316
|
+
}
|
|
317
|
+
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
318
|
+
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
319
|
+
if (words !== null && writeOrd >= 0 && writeOrd < words.length) {
|
|
320
|
+
const seen = Atomics.load(words, writeOrd);
|
|
321
|
+
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
322
|
+
}
|
|
323
|
+
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
324
|
+
for (; ; ) {
|
|
325
|
+
const readers = Atomics.load(header, GRAPH_LOCK);
|
|
326
|
+
if (readers <= FREE) return;
|
|
327
|
+
if (Atomics.compareExchange(header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
328
|
+
if (canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// src/vector/ordinal-filter.ts
|
|
335
|
+
function createOrdinalFilter(slots) {
|
|
336
|
+
return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
|
|
337
|
+
}
|
|
338
|
+
function addToOrdinalFilter(filter, ordinal) {
|
|
339
|
+
const index = ordinal >>> 3;
|
|
340
|
+
if (index >= filter.bits.length) return;
|
|
341
|
+
const bit = 1 << (ordinal & 7);
|
|
342
|
+
if ((filter.bits[index] & bit) !== 0) return;
|
|
343
|
+
filter.bits[index] |= bit;
|
|
344
|
+
filter.count += 1;
|
|
345
|
+
}
|
|
346
|
+
function ordinalFilterHas(filter, ordinal) {
|
|
347
|
+
const index = ordinal >>> 3;
|
|
348
|
+
if (index >= filter.bits.length) return false;
|
|
349
|
+
return (filter.bits[index] & 1 << (ordinal & 7)) !== 0;
|
|
350
|
+
}
|
|
351
|
+
function removeFromOrdinalFilter(filter, ordinal) {
|
|
352
|
+
const index = ordinal >>> 3;
|
|
353
|
+
if (index >= filter.bits.length) return;
|
|
354
|
+
const bit = 1 << (ordinal & 7);
|
|
355
|
+
if ((filter.bits[index] & bit) === 0) return;
|
|
356
|
+
filter.bits[index] &= ~bit;
|
|
357
|
+
filter.count -= 1;
|
|
358
|
+
}
|
|
359
|
+
function* ordinalFilterValues(filter) {
|
|
360
|
+
const bits = filter.bits;
|
|
361
|
+
for (let index = 0; index < bits.length; index++) {
|
|
362
|
+
const byte = bits[index];
|
|
363
|
+
if (byte === 0) continue;
|
|
364
|
+
for (let offset = 0; offset < 8; offset++) {
|
|
365
|
+
if ((byte & 1 << offset) !== 0) yield index << 3 | offset;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// src/vector/filter-ordinals.ts
|
|
371
|
+
var UNRESOLVED_ORDINAL = -1;
|
|
372
|
+
var NO_VECTOR_MARK = -2;
|
|
373
|
+
function freshOrdinals(length) {
|
|
374
|
+
return new Int32Array(length).fill(UNRESOLVED_ORDINAL);
|
|
375
|
+
}
|
|
376
|
+
function mapsOf(owner) {
|
|
377
|
+
owner.vectorOrdinalMaps ??= { bytes: 0, byField: /* @__PURE__ */ new Map() };
|
|
378
|
+
return owner.vectorOrdinalMaps;
|
|
379
|
+
}
|
|
380
|
+
function mapFor(maps, docStore, space, numberingGeneration) {
|
|
381
|
+
const held = maps.byField.get(space.fieldName);
|
|
382
|
+
if (held !== void 0 && held.header === space.header && held.layoutGeneration === space.layoutGeneration && held.numberingGeneration === numberingGeneration) {
|
|
383
|
+
return held;
|
|
384
|
+
}
|
|
385
|
+
const map = {
|
|
386
|
+
header: space.header,
|
|
387
|
+
layoutGeneration: space.layoutGeneration,
|
|
388
|
+
numberingGeneration,
|
|
389
|
+
ordinals: freshOrdinals(docStore.internalIdCapacity())
|
|
390
|
+
};
|
|
391
|
+
maps.bytes += map.ordinals.byteLength - (held?.ordinals.byteLength ?? 0);
|
|
392
|
+
maps.byField.set(space.fieldName, map);
|
|
393
|
+
return map;
|
|
394
|
+
}
|
|
395
|
+
function growTo(maps, map, length) {
|
|
396
|
+
const grown = freshOrdinals(Math.max(length, map.ordinals.length * 2));
|
|
397
|
+
grown.set(map.ordinals);
|
|
398
|
+
maps.bytes += grown.byteLength - map.ordinals.byteLength;
|
|
399
|
+
map.ordinals = grown;
|
|
400
|
+
}
|
|
401
|
+
function ordinalAt(maps, map, internalId, docStore, space) {
|
|
402
|
+
if (internalId >= map.ordinals.length) growTo(maps, map, internalId + 1);
|
|
403
|
+
const held = map.ordinals[internalId];
|
|
404
|
+
if (held >= 0 && space.holdsCurrentOrdinal(held)) return held;
|
|
405
|
+
if (held <= NO_VECTOR_MARK && NO_VECTOR_MARK - held === space.slots) return UNRESOLVED_ORDINAL;
|
|
406
|
+
const docId = docStore.getExternalId(internalId);
|
|
407
|
+
const resolved = docId === void 0 ? void 0 : space.ordinalOf(docId);
|
|
408
|
+
map.ordinals[internalId] = resolved ?? NO_VECTOR_MARK - space.slots;
|
|
409
|
+
return resolved ?? UNRESOLVED_ORDINAL;
|
|
410
|
+
}
|
|
411
|
+
function ordinalFilterForSubsets(subsets, space) {
|
|
412
|
+
const filter = createOrdinalFilter(space.slots);
|
|
413
|
+
for (const subset of subsets) {
|
|
414
|
+
const docStore = subset.owner.docStore;
|
|
415
|
+
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
416
|
+
const resolver = docStore.resolver();
|
|
417
|
+
const maps = mapsOf(subset.owner);
|
|
418
|
+
const map = mapFor(maps, docStore, space, subset.numberingGeneration);
|
|
419
|
+
const bits = subset.bits;
|
|
420
|
+
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
421
|
+
let word = bits[wordIndex];
|
|
422
|
+
if (word === 0) continue;
|
|
423
|
+
const base = wordIndex << 5;
|
|
424
|
+
while (word !== 0) {
|
|
425
|
+
const internalId = base + (Math.clz32(word & -word) ^ 31);
|
|
426
|
+
word &= word - 1;
|
|
427
|
+
if (!resolver.isLive(internalId)) continue;
|
|
428
|
+
const ordinal = ordinalAt(maps, map, internalId, docStore, space);
|
|
429
|
+
if (ordinal < 0 || space.isRemoved(ordinal)) continue;
|
|
430
|
+
addToOrdinalFilter(filter, ordinal);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
return filter;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// src/vector/simd-wasm-binary.ts
|
|
438
|
+
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8DDw4AAQAAAgIDAgIEAgQCAgUDAQABB6QBCgZtZW1vcnkCAAtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
439
|
+
var SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8CEgEDZW52Bm1lbW9yeQIDAYCABAMPDgABAAACAgMCAgQCBAICB5sBCQtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
440
|
+
|
|
441
|
+
// src/vector/simd.ts
|
|
442
|
+
var wasmExports = null;
|
|
443
|
+
var f32View = null;
|
|
444
|
+
function decodeBase64(base64) {
|
|
445
|
+
if (typeof Buffer !== "undefined") {
|
|
446
|
+
return new Uint8Array(Buffer.from(base64, "base64"));
|
|
447
|
+
}
|
|
448
|
+
const binary = atob(base64);
|
|
449
|
+
const bytes = new Uint8Array(binary.length);
|
|
450
|
+
for (let i = 0; i < binary.length; i++) {
|
|
451
|
+
bytes[i] = binary.charCodeAt(i);
|
|
452
|
+
}
|
|
453
|
+
return bytes;
|
|
454
|
+
}
|
|
455
|
+
try {
|
|
456
|
+
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
457
|
+
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
458
|
+
if (typeof WebAssembly !== "undefined" && typeof WebAssembly.validate === "function" && WebAssembly.validate(wasmBuffer)) {
|
|
459
|
+
const module = new WebAssembly.Module(wasmBuffer);
|
|
460
|
+
const instance = new WebAssembly.Instance(module);
|
|
461
|
+
wasmExports = instance.exports;
|
|
462
|
+
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
463
|
+
}
|
|
464
|
+
} catch {
|
|
465
|
+
wasmExports = null;
|
|
466
|
+
f32View = null;
|
|
467
|
+
}
|
|
468
|
+
if (wasmExports === null) markSimdUnavailable();
|
|
469
|
+
var OFFSET_B_BYTES = 16384;
|
|
470
|
+
function ensureMemory(dim) {
|
|
471
|
+
if (!wasmExports || !f32View) return;
|
|
472
|
+
const needed = OFFSET_B_BYTES + dim * 4;
|
|
473
|
+
const currentSize = wasmExports.memory.buffer.byteLength;
|
|
474
|
+
if (needed > currentSize) {
|
|
475
|
+
const pagesNeeded = Math.ceil(needed / 65536);
|
|
476
|
+
const currentPages = currentSize / 65536;
|
|
477
|
+
wasmExports.memory.grow(pagesNeeded - currentPages);
|
|
478
|
+
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
function getF32View() {
|
|
482
|
+
if (!f32View) {
|
|
483
|
+
throw new Error("SIMD memory is not initialized");
|
|
484
|
+
}
|
|
485
|
+
return f32View;
|
|
486
|
+
}
|
|
487
|
+
function copyOneVector(a) {
|
|
488
|
+
getF32View().set(a, 0);
|
|
489
|
+
}
|
|
490
|
+
function copyTwoVectors(a, b) {
|
|
491
|
+
const view = getF32View();
|
|
492
|
+
view.set(a, 0);
|
|
493
|
+
view.set(b, OFFSET_B_BYTES / 4);
|
|
494
|
+
}
|
|
495
|
+
function simdDotProduct(a, b) {
|
|
496
|
+
if (!wasmExports || !f32View) return null;
|
|
497
|
+
ensureMemory(a.length);
|
|
498
|
+
copyTwoVectors(a, b);
|
|
499
|
+
return wasmExports.dot_product(0, OFFSET_B_BYTES, a.length);
|
|
500
|
+
}
|
|
501
|
+
function simdMagnitude(a) {
|
|
502
|
+
if (!wasmExports || !f32View) return null;
|
|
503
|
+
ensureMemory(a.length);
|
|
504
|
+
copyOneVector(a);
|
|
505
|
+
return wasmExports.magnitude(0, a.length);
|
|
506
|
+
}
|
|
507
|
+
function simdSquaredEuclideanDistance(a, b) {
|
|
508
|
+
if (!wasmExports || !f32View) return null;
|
|
509
|
+
ensureMemory(a.length);
|
|
510
|
+
copyTwoVectors(a, b);
|
|
511
|
+
return wasmExports.squared_euclidean_distance(0, OFFSET_B_BYTES, a.length);
|
|
512
|
+
}
|
|
513
|
+
var arenaModule;
|
|
514
|
+
function getArenaModule() {
|
|
515
|
+
if (arenaModule !== void 0) return arenaModule;
|
|
516
|
+
try {
|
|
517
|
+
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
518
|
+
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
519
|
+
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
|
|
520
|
+
arenaModule = null;
|
|
521
|
+
return null;
|
|
522
|
+
}
|
|
523
|
+
arenaModule = new WebAssembly.Module(wasmBuffer);
|
|
524
|
+
} catch {
|
|
525
|
+
arenaModule = null;
|
|
526
|
+
}
|
|
527
|
+
return arenaModule;
|
|
528
|
+
}
|
|
529
|
+
function createArenaSimd() {
|
|
530
|
+
try {
|
|
531
|
+
const module = getArenaModule();
|
|
532
|
+
if (module === null) return null;
|
|
533
|
+
const instance = new WebAssembly.Instance(module);
|
|
534
|
+
const exports$1 = instance.exports;
|
|
535
|
+
if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.osq_dot_nibbles_4x4 !== "function" || typeof exports$1.osq_dot_bits !== "function" || typeof exports$1.osq_dot_bits_pair !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function" || !(exports$1.memory instanceof WebAssembly.Memory)) {
|
|
536
|
+
return null;
|
|
537
|
+
}
|
|
538
|
+
return exports$1;
|
|
539
|
+
} catch {
|
|
540
|
+
return null;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
var sharedMemoryModule;
|
|
544
|
+
function getSharedMemoryModule() {
|
|
545
|
+
if (sharedMemoryModule !== void 0) return sharedMemoryModule;
|
|
546
|
+
try {
|
|
547
|
+
const bytes = decodeBase64(SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64);
|
|
548
|
+
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
549
|
+
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
|
|
550
|
+
sharedMemoryModule = null;
|
|
551
|
+
return null;
|
|
552
|
+
}
|
|
553
|
+
sharedMemoryModule = new WebAssembly.Module(wasmBuffer);
|
|
554
|
+
} catch {
|
|
555
|
+
sharedMemoryModule = null;
|
|
556
|
+
}
|
|
557
|
+
return sharedMemoryModule;
|
|
558
|
+
}
|
|
559
|
+
function createSharedArenaSimd(memory) {
|
|
560
|
+
try {
|
|
561
|
+
const module = getSharedMemoryModule();
|
|
562
|
+
if (module === null) return null;
|
|
563
|
+
const instance = new WebAssembly.Instance(module, { env: { memory } });
|
|
564
|
+
const exports$1 = instance.exports;
|
|
565
|
+
if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.osq_dot_nibbles_4x4 !== "function" || typeof exports$1.osq_dot_bits !== "function" || typeof exports$1.osq_dot_bits_pair !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function") {
|
|
566
|
+
return null;
|
|
567
|
+
}
|
|
568
|
+
return {
|
|
569
|
+
memory,
|
|
570
|
+
dot_product: exports$1.dot_product,
|
|
571
|
+
magnitude: exports$1.magnitude,
|
|
572
|
+
squared_euclidean_distance: exports$1.squared_euclidean_distance,
|
|
573
|
+
dot_u8: exports$1.dot_u8,
|
|
574
|
+
sqdist_u8: exports$1.sqdist_u8,
|
|
575
|
+
osq_dot_nibbles_4x4: exports$1.osq_dot_nibbles_4x4,
|
|
576
|
+
osq_dot_bits: exports$1.osq_dot_bits,
|
|
577
|
+
osq_dot_bits_pair: exports$1.osq_dot_bits_pair
|
|
578
|
+
};
|
|
579
|
+
} catch {
|
|
580
|
+
return null;
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
function arenaFloat32Distance(simd, byteA, byteB, dimension, metric, magnitudeA, magnitudeB) {
|
|
584
|
+
if (metric === "euclidean") {
|
|
585
|
+
return Math.sqrt(simd.squared_euclidean_distance(byteA, byteB, dimension));
|
|
586
|
+
}
|
|
587
|
+
const dot = simd.dot_product(byteA, byteB, dimension);
|
|
588
|
+
if (metric === "dotProduct") return -dot;
|
|
589
|
+
if (magnitudeA === 0 || magnitudeB === 0) return 1;
|
|
590
|
+
return 1 - dot / (magnitudeA * magnitudeB);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
// src/vector/similarity.ts
|
|
594
|
+
function magnitude(v) {
|
|
595
|
+
const result = simdMagnitude(v);
|
|
596
|
+
if (result !== null) return result;
|
|
597
|
+
let sum = 0;
|
|
598
|
+
for (let i = 0; i < v.length; i++) {
|
|
599
|
+
sum += v[i] * v[i];
|
|
600
|
+
}
|
|
601
|
+
return Math.sqrt(sum);
|
|
602
|
+
}
|
|
603
|
+
function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
|
|
604
|
+
if (magA === 0 || magB === 0) return 0;
|
|
605
|
+
return dotProduct(a, b) / (magA * magB);
|
|
606
|
+
}
|
|
607
|
+
function dotProduct(a, b) {
|
|
608
|
+
const result = simdDotProduct(a, b);
|
|
609
|
+
if (result !== null) return result;
|
|
610
|
+
let sum = 0;
|
|
611
|
+
for (let i = 0; i < a.length; i++) {
|
|
612
|
+
sum += a[i] * b[i];
|
|
613
|
+
}
|
|
614
|
+
return sum;
|
|
615
|
+
}
|
|
616
|
+
function euclideanDistance(a, b) {
|
|
617
|
+
return Math.sqrt(squaredEuclideanDistance(a, b));
|
|
618
|
+
}
|
|
619
|
+
function squaredEuclideanDistance(a, b) {
|
|
620
|
+
const result = simdSquaredEuclideanDistance(a, b);
|
|
621
|
+
if (result !== null) return result;
|
|
622
|
+
let sum = 0;
|
|
623
|
+
for (let i = 0; i < a.length; i++) {
|
|
624
|
+
const diff = a[i] - b[i];
|
|
625
|
+
sum += diff * diff;
|
|
626
|
+
}
|
|
627
|
+
return sum;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// src/vector/hnsw/adjacency/views.ts
|
|
631
|
+
var ABSENT = -1;
|
|
632
|
+
function openAdjacency(handles, m, mMax0) {
|
|
633
|
+
return {
|
|
634
|
+
m,
|
|
635
|
+
mMax0,
|
|
636
|
+
level0Stride: mMax0 + 2,
|
|
637
|
+
upperStride: m + 2,
|
|
638
|
+
handles,
|
|
639
|
+
header: handles.header,
|
|
640
|
+
nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
|
|
641
|
+
level0: fixedView(handles.level0, Int32Array),
|
|
642
|
+
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
643
|
+
upper: fixedView(handles.upper, Int32Array),
|
|
644
|
+
incoming: fixedView(handles.incoming, Int32Array)
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
function rebindNodes(adj) {
|
|
648
|
+
adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
|
|
649
|
+
adj.incoming = fixedView(adj.handles.incoming, Int32Array);
|
|
650
|
+
adj.level0 = fixedView(adj.handles.level0, Int32Array);
|
|
651
|
+
adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
|
|
652
|
+
}
|
|
653
|
+
function rebindUpper(adj) {
|
|
654
|
+
adj.upper = fixedView(adj.handles.upper, Int32Array);
|
|
655
|
+
}
|
|
656
|
+
function reachNode(adj, ord) {
|
|
657
|
+
if (ord < adj.nodeLevels.length) return true;
|
|
658
|
+
if (ord >= adj.handles.nodeLevels.byteLength) return false;
|
|
659
|
+
rebindNodes(adj);
|
|
660
|
+
return ord < adj.nodeLevels.length;
|
|
661
|
+
}
|
|
662
|
+
function reachUpper(adj, end) {
|
|
663
|
+
if (end <= adj.upper.length) return true;
|
|
664
|
+
if (end * 4 > adj.handles.upper.byteLength) return false;
|
|
665
|
+
rebindUpper(adj);
|
|
666
|
+
return end <= adj.upper.length;
|
|
667
|
+
}
|
|
668
|
+
function adjacencySlots(adj) {
|
|
669
|
+
return Atomics.load(adj.header, GRAPH_SLOTS);
|
|
670
|
+
}
|
|
671
|
+
function upperUsed(adj) {
|
|
672
|
+
return Atomics.load(adj.header, GRAPH_UPPER_USED);
|
|
673
|
+
}
|
|
674
|
+
function ensureAdjacencyCapacity(adj, needed) {
|
|
675
|
+
if (needed <= adj.nodeLevels.length) return;
|
|
676
|
+
const { handles } = adj;
|
|
677
|
+
if (needed <= handles.nodeLevels.byteLength) {
|
|
678
|
+
rebindNodes(adj);
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
const capacity = Math.max(needed, Math.min(handles.nodeLevels.byteLength * 2, handles.nodeLevels.maxByteLength));
|
|
682
|
+
growBufferToExactly(handles.level0, capacity * adj.level0Stride * 4);
|
|
683
|
+
growBufferToExactly(handles.upperBase, capacity * 4);
|
|
684
|
+
growBufferToExactly(handles.locks, capacity * 4);
|
|
685
|
+
growBufferToExactly(handles.tombstones, capacity);
|
|
686
|
+
growBufferToExactly(handles.incoming, capacity * 4);
|
|
687
|
+
growBufferToExactly(handles.nodeLevels, capacity);
|
|
688
|
+
publishGraphCapacity(handles);
|
|
689
|
+
rebindNodes(adj);
|
|
690
|
+
}
|
|
691
|
+
function growUpperTo(adj, entries) {
|
|
692
|
+
const held = adj.handles.upper.byteLength;
|
|
693
|
+
growBufferTo(adj.handles.upper, entries * 4);
|
|
694
|
+
if (adj.handles.upper.byteLength !== held) publishGraphCapacity(adj.handles);
|
|
695
|
+
}
|
|
696
|
+
function ensureUpperCapacity(adj, entries) {
|
|
697
|
+
growUpperTo(adj, entries);
|
|
698
|
+
rebindUpper(adj);
|
|
699
|
+
}
|
|
700
|
+
function ensureUpperRoom(adj, levels) {
|
|
701
|
+
if (levels < 1) return;
|
|
702
|
+
growUpperTo(adj, Atomics.load(adj.header, GRAPH_UPPER_USED) + levels * adj.upperStride);
|
|
703
|
+
}
|
|
704
|
+
function allocateUpperBlock(adj, levels) {
|
|
705
|
+
const size = levels * adj.upperStride;
|
|
706
|
+
for (; ; ) {
|
|
707
|
+
const offset = Atomics.load(adj.header, GRAPH_UPPER_USED);
|
|
708
|
+
growUpperTo(adj, offset + size);
|
|
709
|
+
if (Atomics.compareExchange(adj.header, GRAPH_UPPER_USED, offset, offset + size) !== offset) continue;
|
|
710
|
+
rebindUpper(adj);
|
|
711
|
+
return offset;
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
function raiseSlots(adj, slots) {
|
|
715
|
+
for (; ; ) {
|
|
716
|
+
const seen = Atomics.load(adj.header, GRAPH_SLOTS);
|
|
717
|
+
if (seen >= slots) return;
|
|
718
|
+
if (Atomics.compareExchange(adj.header, GRAPH_SLOTS, seen, slots) === seen) return;
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
function createNode(adj, ord, maxLayer) {
|
|
722
|
+
ensureAdjacencyCapacity(adj, ord + 1);
|
|
723
|
+
adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) + 1 : 0;
|
|
724
|
+
adj.level0[ord * adj.level0Stride] = 0;
|
|
725
|
+
raiseSlots(adj, ord + 1);
|
|
726
|
+
Atomics.store(adj.nodeLevels, ord, maxLayer + 1);
|
|
727
|
+
}
|
|
728
|
+
function deleteNode(adj, ord) {
|
|
729
|
+
if (ord < 0 || !reachNode(adj, ord)) return;
|
|
730
|
+
Atomics.store(adj.nodeLevels, ord, 0);
|
|
731
|
+
adj.upperBase[ord] = 0;
|
|
732
|
+
adj.level0[ord * adj.level0Stride] = 0;
|
|
733
|
+
Atomics.store(adj.incoming, ord, 0);
|
|
734
|
+
}
|
|
735
|
+
function resetAdjacency(adj) {
|
|
736
|
+
rebindNodes(adj);
|
|
737
|
+
rebindUpper(adj);
|
|
738
|
+
adj.incoming.fill(0);
|
|
739
|
+
adj.nodeLevels.fill(0);
|
|
740
|
+
adj.level0.fill(0);
|
|
741
|
+
adj.upperBase.fill(0);
|
|
742
|
+
adj.upper.fill(0);
|
|
743
|
+
Atomics.store(adj.header, GRAPH_UPPER_USED, 0);
|
|
744
|
+
Atomics.store(adj.header, GRAPH_SLOTS, 0);
|
|
745
|
+
}
|
|
746
|
+
function nodeLevel(adj, ord) {
|
|
747
|
+
if (ord < 0 || !reachNode(adj, ord)) return ABSENT;
|
|
748
|
+
return adj.nodeLevels[ord] - 1;
|
|
749
|
+
}
|
|
750
|
+
function hasNode(adj, ord) {
|
|
751
|
+
return nodeLevel(adj, ord) !== ABSENT;
|
|
752
|
+
}
|
|
753
|
+
function graphBytes(adj) {
|
|
754
|
+
const handles = adj.handles;
|
|
755
|
+
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
// src/vector/hnsw/adjacency/incoming.ts
|
|
759
|
+
function incomingCount(adj, ord) {
|
|
760
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
761
|
+
return Atomics.load(adj.incoming, ord);
|
|
762
|
+
}
|
|
763
|
+
function noteLinked(adj, ord) {
|
|
764
|
+
if (ord < 0 || !reachNode(adj, ord)) return;
|
|
765
|
+
Atomics.add(adj.incoming, ord, 1);
|
|
766
|
+
}
|
|
767
|
+
function noteUnlinked(adj, ord) {
|
|
768
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
769
|
+
for (; ; ) {
|
|
770
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
771
|
+
if (seen <= 0) return 0;
|
|
772
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return seen - 1;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
function releaseUnlessLast(adj, ord) {
|
|
776
|
+
if (ord < 0 || !reachNode(adj, ord)) return true;
|
|
777
|
+
for (; ; ) {
|
|
778
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
779
|
+
if (seen <= 1) return false;
|
|
780
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return true;
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
function holdsNodeAt(adj, ord) {
|
|
784
|
+
return ord >= 0 && ord < adj.nodeLevels.length && adj.nodeLevels[ord] !== 0;
|
|
785
|
+
}
|
|
786
|
+
function countLinksOutOf(adj, ord, step) {
|
|
787
|
+
if (!holdsNodeAt(adj, ord)) return;
|
|
788
|
+
const incoming = adj.incoming;
|
|
789
|
+
const level0 = adj.level0;
|
|
790
|
+
const base = ord * adj.level0Stride;
|
|
791
|
+
const count = level0[base];
|
|
792
|
+
for (let i = 1; i <= count; i++) {
|
|
793
|
+
const neighbour = level0[base + i];
|
|
794
|
+
if (neighbour === ord || !holdsNodeAt(adj, neighbour)) continue;
|
|
795
|
+
if (incoming[neighbour] + step >= 0) incoming[neighbour] += step;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
function countEveryLink(adj, slots) {
|
|
799
|
+
for (let ord = 0; ord < slots; ord++) countLinksOutOf(adj, ord, 1);
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
// src/vector/hnsw/adjacency/neighbors.ts
|
|
803
|
+
function layerArray(adj, layer) {
|
|
804
|
+
return layer === 0 ? adj.level0 : adj.upper;
|
|
805
|
+
}
|
|
806
|
+
function layerBase(adj, ord, layer) {
|
|
807
|
+
const level2 = nodeLevel(adj, ord);
|
|
808
|
+
if (level2 === ABSENT || layer > level2) return ABSENT;
|
|
809
|
+
if (layer === 0) return ord * adj.level0Stride;
|
|
810
|
+
const base = adj.upperBase[ord];
|
|
811
|
+
if (base === 0) return ABSENT;
|
|
812
|
+
const start = base - 1 + (layer - 1) * adj.upperStride;
|
|
813
|
+
return reachUpper(adj, start + adj.upperStride) ? start : ABSENT;
|
|
814
|
+
}
|
|
815
|
+
function neighborCount(adj, ord, layer) {
|
|
816
|
+
const base = layerBase(adj, ord, layer);
|
|
817
|
+
if (base === ABSENT) return 0;
|
|
818
|
+
return layerArray(adj, layer)[base];
|
|
819
|
+
}
|
|
820
|
+
function readNeighbors(adj, ord, layer, out) {
|
|
821
|
+
const base = layerBase(adj, ord, layer);
|
|
822
|
+
if (base === ABSENT) return 0;
|
|
823
|
+
const array = layerArray(adj, layer);
|
|
824
|
+
const count = Math.min(array[base], out.length);
|
|
825
|
+
for (let i = 0; i < count; i++) out[i] = array[base + i + 1];
|
|
826
|
+
return count;
|
|
827
|
+
}
|
|
828
|
+
function collectNeighbors(adj, ord, layer) {
|
|
829
|
+
const base = layerBase(adj, ord, layer);
|
|
830
|
+
if (base === ABSENT) return [];
|
|
831
|
+
const array = layerArray(adj, layer);
|
|
832
|
+
const count = array[base];
|
|
833
|
+
const collected = new Array(count);
|
|
834
|
+
for (let i = 0; i < count; i++) collected[i] = array[base + i + 1];
|
|
835
|
+
return collected;
|
|
836
|
+
}
|
|
837
|
+
function addNeighbor(adj, ord, layer, neighborOrd) {
|
|
838
|
+
const base = layerBase(adj, ord, layer);
|
|
839
|
+
if (base === ABSENT) return;
|
|
840
|
+
const array = layerArray(adj, layer);
|
|
841
|
+
const count = array[base];
|
|
842
|
+
for (let i = 1; i <= count; i++) {
|
|
843
|
+
if (array[base + i] === neighborOrd) return;
|
|
844
|
+
}
|
|
845
|
+
const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
|
|
846
|
+
if (count + 1 > stride - 1) return;
|
|
847
|
+
array[base + count + 1] = neighborOrd;
|
|
848
|
+
array[base] = count + 1;
|
|
849
|
+
if (layer === 0) noteLinked(adj, neighborOrd);
|
|
850
|
+
}
|
|
851
|
+
function removeNeighbor(adj, ord, layer, neighborOrd) {
|
|
852
|
+
const base = layerBase(adj, ord, layer);
|
|
853
|
+
if (base === ABSENT) return;
|
|
854
|
+
const array = layerArray(adj, layer);
|
|
855
|
+
const count = array[base];
|
|
856
|
+
for (let i = 1; i <= count; i++) {
|
|
857
|
+
if (array[base + i] !== neighborOrd) continue;
|
|
858
|
+
for (let j = i; j < count; j++) {
|
|
859
|
+
array[base + j] = array[base + j + 1];
|
|
860
|
+
}
|
|
861
|
+
array[base] = count - 1;
|
|
862
|
+
if (layer === 0) noteUnlinked(adj, neighborOrd);
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
function writeNeighbors(adj, ord, layer, neighbors, count) {
|
|
867
|
+
const base = layerBase(adj, ord, layer);
|
|
868
|
+
if (base === ABSENT) return;
|
|
869
|
+
const array = layerArray(adj, layer);
|
|
870
|
+
const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
|
|
871
|
+
const limit = Math.min(count, stride - 1);
|
|
872
|
+
for (let i = 0; i < limit; i++) {
|
|
873
|
+
array[base + i + 1] = neighbors[i];
|
|
874
|
+
}
|
|
875
|
+
array[base] = limit;
|
|
876
|
+
}
|
|
877
|
+
function holds(neighbors, count, ord) {
|
|
878
|
+
for (let i = 0; i < count; i++) if (neighbors[i] === ord) return true;
|
|
879
|
+
return false;
|
|
880
|
+
}
|
|
881
|
+
function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
882
|
+
const base = layerBase(adj, ord, layer);
|
|
883
|
+
if (base === ABSENT) return;
|
|
884
|
+
if (layer > 0) {
|
|
885
|
+
writeNeighbors(adj, ord, layer, neighbors, count);
|
|
886
|
+
return;
|
|
887
|
+
}
|
|
888
|
+
const array = adj.level0;
|
|
889
|
+
const held = array.slice(base + 1, base + 1 + array[base]);
|
|
890
|
+
const limit = Math.min(count, adj.level0Stride - 1);
|
|
891
|
+
for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
|
|
892
|
+
for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
|
|
893
|
+
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
// src/vector/hnsw/shared.ts
|
|
897
|
+
function entryPointOf(state) {
|
|
898
|
+
return Atomics.load(state.header, GRAPH_ENTRY_POINT);
|
|
899
|
+
}
|
|
900
|
+
function topLayerOf(state) {
|
|
901
|
+
return Atomics.load(state.header, GRAPH_TOP_LAYER);
|
|
902
|
+
}
|
|
903
|
+
function nodeCountOf(state) {
|
|
904
|
+
return Atomics.load(state.header, GRAPH_NODE_COUNT);
|
|
905
|
+
}
|
|
906
|
+
function tombstoneCountOf(state) {
|
|
907
|
+
return Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT);
|
|
908
|
+
}
|
|
909
|
+
function ensureCapacity(state, needed) {
|
|
910
|
+
ensureAdjacencyCapacity(state.adjacency, needed);
|
|
911
|
+
ensureVisited(state, needed);
|
|
912
|
+
}
|
|
913
|
+
function ensureVisited(state, needed) {
|
|
914
|
+
if (needed <= state.visited.length) return;
|
|
915
|
+
let capacity = state.visited.length === 0 ? 16 : state.visited.length;
|
|
916
|
+
while (capacity < needed) capacity *= 2;
|
|
917
|
+
const next = new Uint32Array(capacity);
|
|
918
|
+
next.set(state.visited);
|
|
919
|
+
state.visited = next;
|
|
920
|
+
}
|
|
921
|
+
function nextVisitStamp(state) {
|
|
922
|
+
state.visitStamp++;
|
|
923
|
+
if (state.visitStamp === 4294967295) {
|
|
924
|
+
state.visited.fill(0);
|
|
925
|
+
state.visitStamp = 1;
|
|
926
|
+
}
|
|
927
|
+
return state.visitStamp;
|
|
928
|
+
}
|
|
929
|
+
function reachTombstone(state, ord) {
|
|
930
|
+
if (ord < state.tombstones.length) return true;
|
|
931
|
+
if (ord >= state.adjacency.handles.tombstones.byteLength) return false;
|
|
932
|
+
state.tombstones = fixedView(state.adjacency.handles.tombstones, Uint8Array);
|
|
933
|
+
return ord < state.tombstones.length;
|
|
934
|
+
}
|
|
935
|
+
function isTombstoned(state, ord) {
|
|
936
|
+
return ord >= 0 && reachTombstone(state, ord) && state.tombstones[ord] === 1;
|
|
937
|
+
}
|
|
938
|
+
function nodeExists(state, ord) {
|
|
939
|
+
return hasNode(state.adjacency, ord);
|
|
940
|
+
}
|
|
941
|
+
function nodeMaxLayer(state, ord) {
|
|
942
|
+
return nodeLevel(state.adjacency, ord);
|
|
943
|
+
}
|
|
944
|
+
function toDistance(a, b, magA, magB, metric) {
|
|
945
|
+
switch (metric) {
|
|
946
|
+
case "cosine":
|
|
947
|
+
return 1 - cosineSimilarityWithMagnitudes(a, b, magA, magB);
|
|
948
|
+
case "dotProduct":
|
|
949
|
+
return -dotProduct(a, b);
|
|
950
|
+
case "euclidean":
|
|
951
|
+
return euclideanDistance(a, b);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
function toScore(distance, metric) {
|
|
955
|
+
switch (metric) {
|
|
956
|
+
case "cosine":
|
|
957
|
+
return 1 - distance;
|
|
958
|
+
case "dotProduct":
|
|
959
|
+
return -distance;
|
|
960
|
+
case "euclidean":
|
|
961
|
+
return 1 / (1 + distance);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
function randomLevel(mL) {
|
|
965
|
+
let u = Math.random();
|
|
966
|
+
if (u === 0) u = Number.MIN_VALUE;
|
|
967
|
+
return Math.min(Math.floor(-Math.log(u) * mL), MAX_LAYER_CAP);
|
|
968
|
+
}
|
|
969
|
+
function entryForOrd(state, ord) {
|
|
970
|
+
return state.store.entryForOrdinal(ord);
|
|
971
|
+
}
|
|
972
|
+
function buildsFromCodes(state) {
|
|
973
|
+
return activeQuantizer(state) !== void 0;
|
|
974
|
+
}
|
|
975
|
+
function activeQuantizer(state) {
|
|
976
|
+
const quantizer = state.quantizer;
|
|
977
|
+
if (quantizer === void 0 || !quantizer.isCalibrated()) return void 0;
|
|
978
|
+
return quantizer;
|
|
979
|
+
}
|
|
980
|
+
function nodeDistanceFunction(state, metric) {
|
|
981
|
+
const storeDistance = state.store.pairDistance(metric);
|
|
982
|
+
const quantizer = state.quantizer;
|
|
983
|
+
if (quantizer === void 0 || !buildsFromCodes(state)) return storeDistance;
|
|
984
|
+
const codeDistance = quantizer.pairDistance();
|
|
985
|
+
return (aOrd, bOrd) => {
|
|
986
|
+
const estimated = codeDistance(aOrd, bOrd);
|
|
987
|
+
return estimated === Number.POSITIVE_INFINITY ? storeDistance(aOrd, bOrd) : estimated;
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
function queryDistanceByOrd(state, qVec, qMag, ord, metric) {
|
|
991
|
+
const entry = state.store.entryForOrdinal(ord);
|
|
992
|
+
if (!entry) return Number.POSITIVE_INFINITY;
|
|
993
|
+
return toDistance(qVec, entry.vector, qMag, entry.magnitude, metric);
|
|
994
|
+
}
|
|
995
|
+
function maxConns(state, layer) {
|
|
996
|
+
return layer === 0 ? state.Mmax0 : state.M;
|
|
997
|
+
}
|
|
998
|
+
function addConnection(connections, ord) {
|
|
999
|
+
for (let i = 0; i < connections.length; i++) {
|
|
1000
|
+
if (connections[i] === ord) return;
|
|
1001
|
+
}
|
|
1002
|
+
connections.push(ord);
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// src/vector/native/backend.ts
|
|
1006
|
+
var nativeSearchCoreEnabled = true;
|
|
1007
|
+
function nativeSearchCoreIsEnabled() {
|
|
1008
|
+
return nativeSearchCoreEnabled;
|
|
1009
|
+
}
|
|
1010
|
+
function stopUsingTheNativeSearchCore(error) {
|
|
1011
|
+
if (!nativeSearchCoreEnabled) return;
|
|
1012
|
+
nativeSearchCoreEnabled = false;
|
|
1013
|
+
console.warn(
|
|
1014
|
+
"The native search core raised an error, so this thread searches through WebAssembly from now on:",
|
|
1015
|
+
error instanceof Error ? error.message : String(error)
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
function noteFallback(operation) {
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
// src/vector/vector-store/blocks.ts
|
|
1022
|
+
var SCRATCH_ALIGNMENT_BYTES = 16;
|
|
1023
|
+
var BLOCK_MAX_BYTES = MAX_WASM_PAGES * WASM_PAGE_BYTES;
|
|
1024
|
+
function alignUp(bytes) {
|
|
1025
|
+
return Math.ceil(bytes / SCRATCH_ALIGNMENT_BYTES) * SCRATCH_ALIGNMENT_BYTES;
|
|
1026
|
+
}
|
|
1027
|
+
function computeVectorBlockLayout(dimension, blockBytes2 = BLOCK_MAX_BYTES) {
|
|
1028
|
+
const scratchStride = alignUp(dimension * 4);
|
|
1029
|
+
const slotsOffset = alignUp(VECTOR_SCRATCH_SLOTS * scratchStride);
|
|
1030
|
+
const slotStride = alignUp(dimension * 4);
|
|
1031
|
+
return {
|
|
1032
|
+
dimension,
|
|
1033
|
+
scratchStride,
|
|
1034
|
+
slotsOffset,
|
|
1035
|
+
slotStride,
|
|
1036
|
+
capacity: Math.max(1, Math.floor((Math.min(blockBytes2, BLOCK_MAX_BYTES) - slotsOffset) / slotStride))
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
function initialBlockBytes(layout) {
|
|
1040
|
+
return layout.slotsOffset + VECTOR_BLOCK_INITIAL_SLOTS * layout.slotStride;
|
|
1041
|
+
}
|
|
1042
|
+
function pagesFor(bytes) {
|
|
1043
|
+
return Math.ceil(bytes / WASM_PAGE_BYTES);
|
|
1044
|
+
}
|
|
1045
|
+
function createSharedMemory(pages) {
|
|
1046
|
+
if (!sharedMemoryAvailable() || typeof WebAssembly === "undefined") return null;
|
|
1047
|
+
try {
|
|
1048
|
+
const memory = new WebAssembly.Memory({ initial: pages, maximum: MAX_WASM_PAGES, shared: true });
|
|
1049
|
+
return createSharedArenaSimd(memory) === null ? null : memory;
|
|
1050
|
+
} catch {
|
|
1051
|
+
return null;
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
function createVectorBlock(layout) {
|
|
1055
|
+
const initialBytes = initialBlockBytes(layout);
|
|
1056
|
+
const shared = createSharedMemory(pagesFor(initialBytes));
|
|
1057
|
+
if (shared !== null) return { layout, storage: { kind: "shared-memory", memory: shared } };
|
|
1058
|
+
const simd = createArenaSimd();
|
|
1059
|
+
if (simd !== null) {
|
|
1060
|
+
const missing = pagesFor(initialBytes) - simd.memory.buffer.byteLength / WASM_PAGE_BYTES;
|
|
1061
|
+
if (missing > 0) simd.memory.grow(missing);
|
|
1062
|
+
return { layout, storage: { kind: "private-memory", simd } };
|
|
1063
|
+
}
|
|
1064
|
+
return { layout, storage: { kind: "bytes", buffer: createGrowableBuffer(initialBytes, fullBlockBytes(layout)) } };
|
|
1065
|
+
}
|
|
1066
|
+
function fullBlockBytes(layout) {
|
|
1067
|
+
return Math.min(BLOCK_MAX_BYTES, layout.slotsOffset + layout.capacity * layout.slotStride);
|
|
1068
|
+
}
|
|
1069
|
+
function growMemoryTo(memory, bytes, layout) {
|
|
1070
|
+
const have = memory.buffer.byteLength;
|
|
1071
|
+
if (have >= bytes) return;
|
|
1072
|
+
memory.grow(pagesFor(nextGrowthTarget(have, bytes, fullBlockBytes(layout))) - have / WASM_PAGE_BYTES);
|
|
1073
|
+
}
|
|
1074
|
+
function ensureBlockSlots(handle, slots) {
|
|
1075
|
+
const bytes = handle.layout.slotsOffset + Math.min(slots, handle.layout.capacity) * handle.layout.slotStride;
|
|
1076
|
+
const storage = handle.storage;
|
|
1077
|
+
if (storage.kind === "bytes") {
|
|
1078
|
+
growBufferTo(storage.buffer, bytes);
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
1082
|
+
}
|
|
1083
|
+
function blockBuffer(handle) {
|
|
1084
|
+
const storage = handle.storage;
|
|
1085
|
+
if (storage.kind === "bytes") return storage.buffer;
|
|
1086
|
+
return storage.kind === "shared-memory" ? storage.memory.buffer : storage.simd.memory.buffer;
|
|
1087
|
+
}
|
|
1088
|
+
var NOTHING_STAGED = -1;
|
|
1089
|
+
var QUERY_STAGED = -2;
|
|
1090
|
+
function openVectorBlock(handle, threadSlot) {
|
|
1091
|
+
const { layout, storage } = handle;
|
|
1092
|
+
const simd = storage.kind === "shared-memory" ? createSharedArenaSimd(storage.memory) : storage.kind === "private-memory" ? storage.simd : null;
|
|
1093
|
+
const hasScratch = simd !== null && threadSlot >= 0 && threadSlot < VECTOR_SCRATCH_SLOTS;
|
|
1094
|
+
const scratchSlot = hasScratch ? threadSlot : 0;
|
|
1095
|
+
let float32View;
|
|
1096
|
+
let byteView;
|
|
1097
|
+
let dataView;
|
|
1098
|
+
function rebindViews() {
|
|
1099
|
+
const buffer = blockBuffer(handle);
|
|
1100
|
+
float32View = fixedView(buffer, Float32Array);
|
|
1101
|
+
byteView = fixedView(buffer, Uint8Array);
|
|
1102
|
+
dataView = new DataView(buffer, 0, byteView.length);
|
|
1103
|
+
}
|
|
1104
|
+
rebindViews();
|
|
1105
|
+
return {
|
|
1106
|
+
handle,
|
|
1107
|
+
simd,
|
|
1108
|
+
hasScratch,
|
|
1109
|
+
scratchByteOffset: scratchSlot * layout.scratchStride,
|
|
1110
|
+
stagedOrdinal: NOTHING_STAGED,
|
|
1111
|
+
float32(neededLength) {
|
|
1112
|
+
if (neededLength > float32View.length) rebindViews();
|
|
1113
|
+
return float32View;
|
|
1114
|
+
},
|
|
1115
|
+
bytes(neededLength) {
|
|
1116
|
+
if (neededLength > byteView.length) rebindViews();
|
|
1117
|
+
return byteView;
|
|
1118
|
+
},
|
|
1119
|
+
data(neededLength) {
|
|
1120
|
+
if (neededLength > byteView.length) rebindViews();
|
|
1121
|
+
return dataView;
|
|
1122
|
+
}
|
|
1123
|
+
};
|
|
1124
|
+
}
|
|
1125
|
+
function slotByteOffset(layout, localOrdinal) {
|
|
1126
|
+
return layout.slotsOffset + localOrdinal * layout.slotStride;
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// src/vector/osq/estimate.ts
|
|
1130
|
+
var POPCOUNT = new Uint8Array(256);
|
|
1131
|
+
for (let value = 0; value < 256; value++) {
|
|
1132
|
+
let count = 0;
|
|
1133
|
+
let rest = value;
|
|
1134
|
+
while (rest !== 0) {
|
|
1135
|
+
count += rest & 1;
|
|
1136
|
+
rest >>= 1;
|
|
1137
|
+
}
|
|
1138
|
+
POPCOUNT[value] = count;
|
|
1139
|
+
}
|
|
1140
|
+
var LEVEL_BIT_MASK = { 1: 255, 2: 85 };
|
|
1141
|
+
var STAGED_QUERY_PLANES = 4;
|
|
1142
|
+
function osqNibblesByByteLevels(document2, documentOffset, levels, bytes) {
|
|
1143
|
+
let total = 0;
|
|
1144
|
+
for (let n = 0; n < bytes; n++) {
|
|
1145
|
+
const packed = document2[documentOffset + n];
|
|
1146
|
+
total += (packed & 15) * levels[2 * n] + (packed >> 4) * levels[2 * n + 1];
|
|
1147
|
+
}
|
|
1148
|
+
return total;
|
|
1149
|
+
}
|
|
1150
|
+
function osqNarrowPairProducts(a, aOffset, b, bOffset, bits, bytes) {
|
|
1151
|
+
const mask = LEVEL_BIT_MASK[bits];
|
|
1152
|
+
let total = 0;
|
|
1153
|
+
for (let j = 0; j < bits; j++) {
|
|
1154
|
+
for (let l = 0; l < bits; l++) {
|
|
1155
|
+
let bitsSet = 0;
|
|
1156
|
+
for (let n = 0; n < bytes; n++) bitsSet += POPCOUNT[a[aOffset + n] >> j & b[bOffset + n] >> l & mask];
|
|
1157
|
+
total += 2 ** (j + l) * bitsSet;
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
return total;
|
|
1161
|
+
}
|
|
1162
|
+
function osqNibbleProducts(a, aOffset, b, bOffset, bytes) {
|
|
1163
|
+
let total = 0;
|
|
1164
|
+
for (let n = 0; n < bytes; n++) {
|
|
1165
|
+
const left = a[aOffset + n];
|
|
1166
|
+
const right = b[bOffset + n];
|
|
1167
|
+
total += (left & 15) * (right & 15) + (left >> 4) * (right >> 4);
|
|
1168
|
+
}
|
|
1169
|
+
return total;
|
|
1170
|
+
}
|
|
1171
|
+
function twoBitStagedProducts(document2, documentOffset, streams, streamsOffset, streamBytes) {
|
|
1172
|
+
let total = 0;
|
|
1173
|
+
for (let n = 0; n < streamBytes; n++) {
|
|
1174
|
+
const packed = document2[documentOffset + n];
|
|
1175
|
+
for (let k = 0; k < STAGED_QUERY_PLANES; k++) {
|
|
1176
|
+
total += (packed >> 2 * k & 3) * streams[streamsOffset + k * streamBytes + n];
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
return total;
|
|
1180
|
+
}
|
|
1181
|
+
function osqStagedQueryProducts(document2, documentOffset, documentBits, staged, stagedOffset, codeBytes) {
|
|
1182
|
+
if (documentBits === 2) return twoBitStagedProducts(document2, documentOffset, staged, stagedOffset, codeBytes);
|
|
1183
|
+
let total = 0;
|
|
1184
|
+
for (let p = 0; p < STAGED_QUERY_PLANES; p++) {
|
|
1185
|
+
const plane = stagedOffset + p * codeBytes;
|
|
1186
|
+
let bitsSet = 0;
|
|
1187
|
+
for (let n = 0; n < codeBytes; n++) bitsSet += POPCOUNT[document2[documentOffset + n] & staged[plane + n]];
|
|
1188
|
+
total += 2 ** p * bitsSet;
|
|
1189
|
+
}
|
|
1190
|
+
return total;
|
|
1191
|
+
}
|
|
1192
|
+
function osqByteLevelProducts(document2, documentOffset, query, queryOffset, dimension) {
|
|
1193
|
+
let total = 0;
|
|
1194
|
+
for (let i = 0; i < dimension; i++) total += document2[documentOffset + i] * query[queryOffset + i];
|
|
1195
|
+
return total;
|
|
1196
|
+
}
|
|
1197
|
+
function osqEstimate(products, document2, documentBits, query, queryBits, dimension, centroidDot, metric) {
|
|
1198
|
+
const documentStep = (document2.upper - document2.lower) / OSQ_STEPS[documentBits];
|
|
1199
|
+
const queryStep = (query.upper - query.lower) / OSQ_STEPS[queryBits];
|
|
1200
|
+
const centred = document2.lower * query.lower * dimension + query.lower * documentStep * document2.sum + document2.lower * queryStep * query.sum + documentStep * queryStep * products;
|
|
1201
|
+
if (metric === "euclidean") return Math.sqrt(Math.max(0, document2.correction + query.correction - 2 * centred));
|
|
1202
|
+
const similarity = centred + document2.correction + query.correction - centroidDot;
|
|
1203
|
+
if (metric === "cosine") return Math.min(Math.max(similarity, -1), 1);
|
|
1204
|
+
return similarity;
|
|
1205
|
+
}
|
|
1206
|
+
function osqDistance(estimate, metric) {
|
|
1207
|
+
switch (metric) {
|
|
1208
|
+
case "cosine":
|
|
1209
|
+
return 1 - estimate;
|
|
1210
|
+
case "dotProduct":
|
|
1211
|
+
return -estimate;
|
|
1212
|
+
case "euclidean":
|
|
1213
|
+
return estimate;
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
function osqQueryBits(bits) {
|
|
1217
|
+
return OSQ_QUERY_BITS[bits];
|
|
1218
|
+
}
|
|
1219
|
+
function osqPlacementQueryBits(bits) {
|
|
1220
|
+
return OSQ_PLACEMENT_QUERY_BITS[bits];
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
// src/vector/vector-store/code-blocks.ts
|
|
1224
|
+
function computeCodeBlockLayout(dimension, bits) {
|
|
1225
|
+
const queryBits = osqQueryBits(bits);
|
|
1226
|
+
const codeBytes = osqCodeBytes(dimension, bits);
|
|
1227
|
+
const queryBytes = osqStagedQueryBytes(dimension, bits);
|
|
1228
|
+
const scratchStride = alignUp(queryBytes);
|
|
1229
|
+
const slotsOffset = alignUp(VECTOR_SCRATCH_SLOTS * scratchStride);
|
|
1230
|
+
const slotStride = osqRecordBytes(dimension, bits);
|
|
1231
|
+
return {
|
|
1232
|
+
dimension,
|
|
1233
|
+
bits,
|
|
1234
|
+
queryBits,
|
|
1235
|
+
codeBytes,
|
|
1236
|
+
queryBytes,
|
|
1237
|
+
scratchStride,
|
|
1238
|
+
slotsOffset,
|
|
1239
|
+
slotStride,
|
|
1240
|
+
capacity: Math.floor((BLOCK_MAX_BYTES - slotsOffset) / slotStride)
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
// src/vector/vector-store/handles.ts
|
|
1245
|
+
var STORE_SLOTS = 0;
|
|
1246
|
+
var STORE_LIVE_COUNT = 1;
|
|
1247
|
+
var STORE_BLOCK_COUNT = 2;
|
|
1248
|
+
var STORE_CALIBRATED = 3;
|
|
1249
|
+
var STORE_CODE_COUNT = 4;
|
|
1250
|
+
var STORE_DOC_ID_BYTES = 5;
|
|
1251
|
+
var STORE_CODE_BLOCK_COUNT = 6;
|
|
1252
|
+
var STORE_CALIBRATION_GENERATION = 7;
|
|
1253
|
+
var STORE_RELEASED_VECTORS_TO_DISK = 8;
|
|
1254
|
+
var STORE_UNPLACED_COUNT = 9;
|
|
1255
|
+
var STORE_REMOVED_COUNT = 10;
|
|
1256
|
+
var STORE_ORDINAL_LAYOUT_GENERATION = 11;
|
|
1257
|
+
var STORE_HEADER_WORDS = 16;
|
|
1258
|
+
var IN_MEMORY = -1;
|
|
1259
|
+
function resetStoreHeader(header) {
|
|
1260
|
+
Atomics.store(header, STORE_SLOTS, 0);
|
|
1261
|
+
Atomics.store(header, STORE_RELEASED_VECTORS_TO_DISK, 0);
|
|
1262
|
+
Atomics.store(header, STORE_LIVE_COUNT, 0);
|
|
1263
|
+
Atomics.store(header, STORE_CODE_COUNT, 0);
|
|
1264
|
+
Atomics.store(header, STORE_CALIBRATED, 0);
|
|
1265
|
+
Atomics.add(header, STORE_CALIBRATION_GENERATION, 1);
|
|
1266
|
+
Atomics.add(header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
1267
|
+
}
|
|
1268
|
+
function fixedInt32(words) {
|
|
1269
|
+
return new Int32Array(createFixedBuffer(words * 4), 0, words);
|
|
1270
|
+
}
|
|
1271
|
+
function fixedFloat32(length) {
|
|
1272
|
+
return new Float32Array(createFixedBuffer(length * 4), 0, length);
|
|
1273
|
+
}
|
|
1274
|
+
function unplacedChunkCount(ordinals) {
|
|
1275
|
+
return Math.floor(ordinals / UNPLACED_FLAG_CHUNK_ORDINALS) + 1;
|
|
1276
|
+
}
|
|
1277
|
+
function perOrdinal2(bytesPerOrdinal) {
|
|
1278
|
+
return createGrowableBuffer(
|
|
1279
|
+
VECTOR_STORE_INITIAL_CAPACITY * bytesPerOrdinal,
|
|
1280
|
+
(MAX_VECTOR_ORDINALS + 1) * bytesPerOrdinal
|
|
1281
|
+
);
|
|
1282
|
+
}
|
|
1283
|
+
function inMemoryTable() {
|
|
1284
|
+
const buffer = perOrdinal2(4);
|
|
1285
|
+
new Int32Array(buffer).fill(IN_MEMORY);
|
|
1286
|
+
return buffer;
|
|
1287
|
+
}
|
|
1288
|
+
function createSharedVectorStoreHandles(dimension, codeBits, blockBytes2 = BLOCK_MAX_BYTES) {
|
|
1289
|
+
return {
|
|
1290
|
+
dimension,
|
|
1291
|
+
codeBits,
|
|
1292
|
+
layout: computeVectorBlockLayout(dimension, blockBytes2),
|
|
1293
|
+
codeLayout: codeBits === null ? null : computeCodeBlockLayout(dimension, codeBits),
|
|
1294
|
+
layoutRevision: 0,
|
|
1295
|
+
header: fixedInt32(STORE_HEADER_WORDS),
|
|
1296
|
+
blocks: [],
|
|
1297
|
+
codeBlocks: [],
|
|
1298
|
+
vectorFiles: [],
|
|
1299
|
+
magnitudes: perOrdinal2(8),
|
|
1300
|
+
present: perOrdinal2(1),
|
|
1301
|
+
unplaced: perOrdinal2(1),
|
|
1302
|
+
unplacedChunks: createGrowableBuffer(4, unplacedChunkCount(MAX_VECTOR_ORDINALS) * 4),
|
|
1303
|
+
removed: perOrdinal2(1),
|
|
1304
|
+
partitions: perOrdinal2(4),
|
|
1305
|
+
docIdBytes: createGrowableBuffer(VECTOR_STORE_INITIAL_CAPACITY * 16, MAX_DOC_ID_TABLE_BYTES),
|
|
1306
|
+
docIdOffsets: perOrdinal2(4),
|
|
1307
|
+
codePresent: perOrdinal2(1),
|
|
1308
|
+
diskFile: inMemoryTable(),
|
|
1309
|
+
diskOffset: perOrdinal2(4),
|
|
1310
|
+
centroid: fixedFloat32(dimension)
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1313
|
+
function sharedVectorStoreBytes(handles) {
|
|
1314
|
+
let bytes = handles.header.byteLength + handles.centroid.byteLength;
|
|
1315
|
+
for (const block of handles.blocks) {
|
|
1316
|
+
if (block !== null) bytes += blockBuffer(block).byteLength;
|
|
1317
|
+
}
|
|
1318
|
+
for (const block of handles.codeBlocks) bytes += blockBuffer(block).byteLength;
|
|
1319
|
+
bytes += handles.magnitudes.byteLength;
|
|
1320
|
+
bytes += handles.present.byteLength;
|
|
1321
|
+
bytes += handles.unplaced.byteLength;
|
|
1322
|
+
bytes += handles.unplacedChunks.byteLength;
|
|
1323
|
+
bytes += handles.removed.byteLength;
|
|
1324
|
+
bytes += handles.partitions.byteLength;
|
|
1325
|
+
bytes += handles.docIdBytes.byteLength;
|
|
1326
|
+
bytes += handles.docIdOffsets.byteLength;
|
|
1327
|
+
bytes += handles.codePresent.byteLength;
|
|
1328
|
+
bytes += handles.diskFile.byteLength;
|
|
1329
|
+
bytes += handles.diskOffset.byteLength;
|
|
1330
|
+
return bytes;
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
// src/vector/native/types.ts
|
|
1334
|
+
var NATIVE_OK = 0;
|
|
1335
|
+
var NATIVE_NEEDS_ROOM = 3;
|
|
1336
|
+
|
|
1337
|
+
// src/vector/native/store.ts
|
|
1338
|
+
var METRIC_CODES2 = { cosine: 0, dotProduct: 1, euclidean: 2 };
|
|
1339
|
+
var attachedStores = /* @__PURE__ */ new WeakMap();
|
|
1340
|
+
function nativeMetricCode(metric) {
|
|
1341
|
+
return METRIC_CODES2[metric];
|
|
1342
|
+
}
|
|
1343
|
+
function isUsable(attached) {
|
|
1344
|
+
return attached.handle !== null;
|
|
1345
|
+
}
|
|
1346
|
+
function detachNativeStore(store) {
|
|
1347
|
+
const attached = attachedStores.get(store);
|
|
1348
|
+
if (attached === void 0) return;
|
|
1349
|
+
attachedStores.delete(store);
|
|
1350
|
+
if (attached.handle === null) return;
|
|
1351
|
+
try {
|
|
1352
|
+
attached.core.detachStore(attached.handle);
|
|
1353
|
+
} catch (error) {
|
|
1354
|
+
stopUsingTheNativeSearchCore(error);
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
function attachedNativeStore(store) {
|
|
1358
|
+
const attached = attachedStores.get(store);
|
|
1359
|
+
return attached !== void 0 && isUsable(attached) ? attached : null;
|
|
1360
|
+
}
|
|
1361
|
+
function nativeStoreScratchBytes(store) {
|
|
1362
|
+
const attached = attachedNativeStore(store);
|
|
1363
|
+
return attached === null ? 0 : attached.ordinals.byteLength + attached.distances.byteLength;
|
|
1364
|
+
}
|
|
1365
|
+
function nativeStoreFor(store) {
|
|
1366
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
1367
|
+
return null;
|
|
1368
|
+
}
|
|
1369
|
+
function reserveOrdinals(store, count) {
|
|
1370
|
+
if (store.ordinals.length >= count) return;
|
|
1371
|
+
store.ordinals = new Int32Array(count);
|
|
1372
|
+
store.distances = new Float64Array(count);
|
|
1373
|
+
}
|
|
1374
|
+
function nativeScores(store, query, metric, count) {
|
|
1375
|
+
if (count === 0) return store.distances;
|
|
1376
|
+
let status;
|
|
1377
|
+
try {
|
|
1378
|
+
status = store.core.score(store.handle, query, nativeMetricCode(metric), store.ordinals, count, store.distances);
|
|
1379
|
+
} catch (error) {
|
|
1380
|
+
stopUsingTheNativeSearchCore(error);
|
|
1381
|
+
return null;
|
|
1382
|
+
}
|
|
1383
|
+
return status === NATIVE_OK ? store.distances : null;
|
|
1384
|
+
}
|
|
1385
|
+
function nativeScoresOf(handles, query, metric, ordinals) {
|
|
1386
|
+
const store = nativeStoreFor();
|
|
1387
|
+
if (store === null) return null;
|
|
1388
|
+
reserveOrdinals(store, ordinals.length);
|
|
1389
|
+
store.ordinals.set(ordinals);
|
|
1390
|
+
return nativeScores(store, query, metric, ordinals.length);
|
|
1391
|
+
}
|
|
1392
|
+
function workOnRecords(handles, ordinals, work) {
|
|
1393
|
+
if (ordinals.length === 0) return true;
|
|
1394
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1395
|
+
const store = nativeStoreFor();
|
|
1396
|
+
if (store === null) return false;
|
|
1397
|
+
let status;
|
|
1398
|
+
try {
|
|
1399
|
+
status = work(store);
|
|
1400
|
+
} catch (error) {
|
|
1401
|
+
stopUsingTheNativeSearchCore(error);
|
|
1402
|
+
return false;
|
|
1403
|
+
}
|
|
1404
|
+
if (status === NATIVE_OK) return true;
|
|
1405
|
+
if (status !== NATIVE_NEEDS_ROOM) return false;
|
|
1406
|
+
detachNativeStore(handles);
|
|
1407
|
+
}
|
|
1408
|
+
return false;
|
|
1409
|
+
}
|
|
1410
|
+
function nativeMeasure(handles, metric, ordinals, totals) {
|
|
1411
|
+
return workOnRecords(
|
|
1412
|
+
handles,
|
|
1413
|
+
ordinals,
|
|
1414
|
+
(store) => store.core.measure(store.handle, nativeMetricCode(metric), ordinals, ordinals.length, totals)
|
|
1415
|
+
);
|
|
1416
|
+
}
|
|
1417
|
+
function nativeQuantise(handles, metric, ordinals) {
|
|
1418
|
+
if (Atomics.load(handles.header, STORE_CALIBRATED) !== 1) return false;
|
|
1419
|
+
return workOnRecords(
|
|
1420
|
+
handles,
|
|
1421
|
+
ordinals,
|
|
1422
|
+
(store) => store.core.quantise(store.handle, nativeMetricCode(metric), ordinals, ordinals.length)
|
|
1423
|
+
);
|
|
1424
|
+
}
|
|
1425
|
+
var WAKE_ENTRY_LOCK = -1;
|
|
1426
|
+
var attachedGraphs = /* @__PURE__ */ new WeakMap();
|
|
1427
|
+
var workspaceOwner = null;
|
|
1428
|
+
function nativeScratchBytes(state, field) {
|
|
1429
|
+
const perField = field.store.ordinals.byteLength + field.store.distances.byteLength + field.wakes.byteLength;
|
|
1430
|
+
const owner = workspaceOwner?.deref();
|
|
1431
|
+
if (owner !== void 0 && owner !== state) return perField;
|
|
1432
|
+
workspaceOwner = new WeakRef(state);
|
|
1433
|
+
return perField + field.core.workspaceBytes();
|
|
1434
|
+
}
|
|
1435
|
+
function attachedNativeField(state) {
|
|
1436
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
1437
|
+
return attachedGraphs.get(state)?.field ?? null;
|
|
1438
|
+
}
|
|
1439
|
+
function nativeFieldFor(state) {
|
|
1440
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
1441
|
+
return null;
|
|
1442
|
+
}
|
|
1443
|
+
function wakeSleepingThreads(state, field) {
|
|
1444
|
+
const count = field.wakes[0];
|
|
1445
|
+
if (count === 0) return;
|
|
1446
|
+
field.wakes[0] = 0;
|
|
1447
|
+
for (let i = 1; i <= count; i++) {
|
|
1448
|
+
const ordinal = field.wakes[i];
|
|
1449
|
+
if (ordinal === WAKE_ENTRY_LOCK) {
|
|
1450
|
+
Atomics.notify(state.header, GRAPH_ENTRY_LOCK);
|
|
1451
|
+
continue;
|
|
1452
|
+
}
|
|
1453
|
+
if (ordinal < 0 || ordinal >= Atomics.load(state.header, GRAPH_SLOTS)) continue;
|
|
1454
|
+
Atomics.notify(state.locks.words(ordinal), ordinal);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
// src/vector/native/writes.ts
|
|
1459
|
+
var CALL_RAISED = -1;
|
|
1460
|
+
function writeToTheGraph(state, field, write) {
|
|
1461
|
+
let status;
|
|
1462
|
+
try {
|
|
1463
|
+
status = write();
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
stopUsingTheNativeSearchCore(error);
|
|
1466
|
+
return CALL_RAISED;
|
|
1467
|
+
}
|
|
1468
|
+
wakeSleepingThreads(state, field);
|
|
1469
|
+
return status;
|
|
1470
|
+
}
|
|
1471
|
+
function nativePlace(state, field, metric, ordinal, topLayer, graphHeldAlone) {
|
|
1472
|
+
const status = writeToTheGraph(
|
|
1473
|
+
state,
|
|
1474
|
+
field,
|
|
1475
|
+
() => field.core.place(
|
|
1476
|
+
field.graph,
|
|
1477
|
+
field.store.handle,
|
|
1478
|
+
nativeMetricCode(metric),
|
|
1479
|
+
ordinal,
|
|
1480
|
+
topLayer,
|
|
1481
|
+
graphHeldAlone ? 1 : 0,
|
|
1482
|
+
state.locks.threadSlot,
|
|
1483
|
+
field.wakes
|
|
1484
|
+
)
|
|
1485
|
+
);
|
|
1486
|
+
if (status === NATIVE_OK) return "placed";
|
|
1487
|
+
return status === NATIVE_NEEDS_ROOM ? "needsRoom" : "failed";
|
|
1488
|
+
}
|
|
1489
|
+
function nativeRemove(state, field, ordinal) {
|
|
1490
|
+
const remove = () => field.core.remove(field.graph, ordinal, state.locks.threadSlot, field.wakes);
|
|
1491
|
+
return writeToTheGraph(state, field, remove) === NATIVE_OK;
|
|
1492
|
+
}
|
|
1493
|
+
function nativeCompact(state, field, metric) {
|
|
1494
|
+
const compact = () => field.core.compact(field.graph, field.store.handle, nativeMetricCode(metric), state.locks.threadSlot, field.wakes);
|
|
1495
|
+
return writeToTheGraph(state, field, compact) === NATIVE_OK;
|
|
1496
|
+
}
|
|
1497
|
+
function nativeConnect(state, field, metric, nearby) {
|
|
1498
|
+
const connect = () => field.core.connectUnreached(
|
|
1499
|
+
field.graph,
|
|
1500
|
+
field.store.handle,
|
|
1501
|
+
nativeMetricCode(metric),
|
|
1502
|
+
nearby,
|
|
1503
|
+
state.locks.threadSlot,
|
|
1504
|
+
field.wakes
|
|
1505
|
+
);
|
|
1506
|
+
return writeToTheGraph(state, field, connect) === NATIVE_OK;
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
// src/vector/hnsw/workspace.ts
|
|
1510
|
+
function createList(capacity) {
|
|
1511
|
+
return { ords: new Int32Array(capacity), distances: new Float64Array(capacity), size: 0 };
|
|
1512
|
+
}
|
|
1513
|
+
function createHeap(greatestFirst) {
|
|
1514
|
+
return { ...createList(INITIAL_LIST_CAPACITY), greatestFirst, topOrd: -1, topDistance: 0 };
|
|
1515
|
+
}
|
|
1516
|
+
function createHNSWWorkspace() {
|
|
1517
|
+
return {
|
|
1518
|
+
frontier: createHeap(false),
|
|
1519
|
+
found: createHeap(true),
|
|
1520
|
+
entryPoints: new Int32Array(INITIAL_LIST_CAPACITY),
|
|
1521
|
+
entryPointCount: 0,
|
|
1522
|
+
traversal: createList(INITIAL_LIST_CAPACITY),
|
|
1523
|
+
working: createList(INITIAL_LIST_CAPACITY),
|
|
1524
|
+
linkSelections: [],
|
|
1525
|
+
pruneCandidates: createList(INITIAL_LIST_CAPACITY),
|
|
1526
|
+
pruneSelection: createList(INITIAL_LIST_CAPACITY),
|
|
1527
|
+
repairCandidates: createList(INITIAL_LIST_CAPACITY),
|
|
1528
|
+
repairSelection: createList(INITIAL_LIST_CAPACITY),
|
|
1529
|
+
droppedLinks: createList(INITIAL_LIST_CAPACITY)
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1532
|
+
function linkSelectionsFor(workspace, layers) {
|
|
1533
|
+
const selections = workspace.linkSelections;
|
|
1534
|
+
while (selections.length < layers) selections.push(createList(INITIAL_LIST_CAPACITY));
|
|
1535
|
+
return selections;
|
|
1536
|
+
}
|
|
1537
|
+
function ensureListCapacity(list, needed) {
|
|
1538
|
+
if (needed <= list.ords.length) return;
|
|
1539
|
+
let capacity = list.ords.length === 0 ? INITIAL_LIST_CAPACITY : list.ords.length;
|
|
1540
|
+
while (capacity < needed) capacity *= 2;
|
|
1541
|
+
const ords = new Int32Array(capacity);
|
|
1542
|
+
ords.set(list.ords);
|
|
1543
|
+
list.ords = ords;
|
|
1544
|
+
const distances = new Float64Array(capacity);
|
|
1545
|
+
distances.set(list.distances);
|
|
1546
|
+
list.distances = distances;
|
|
1547
|
+
}
|
|
1548
|
+
function appendToList(list, ord, distance) {
|
|
1549
|
+
ensureListCapacity(list, list.size + 1);
|
|
1550
|
+
list.ords[list.size] = ord;
|
|
1551
|
+
list.distances[list.size] = distance;
|
|
1552
|
+
list.size += 1;
|
|
1553
|
+
}
|
|
1554
|
+
function copyList(source, target) {
|
|
1555
|
+
ensureListCapacity(target, source.size);
|
|
1556
|
+
target.ords.set(source.ords.subarray(0, source.size));
|
|
1557
|
+
target.distances.set(source.distances.subarray(0, source.size));
|
|
1558
|
+
target.size = source.size;
|
|
1559
|
+
}
|
|
1560
|
+
function sortListByDistance(list) {
|
|
1561
|
+
const { ords, distances } = list;
|
|
1562
|
+
for (let i = 1; i < list.size; i++) {
|
|
1563
|
+
const ord = ords[i];
|
|
1564
|
+
const distance = distances[i];
|
|
1565
|
+
let j = i - 1;
|
|
1566
|
+
while (j >= 0 && distances[j] > distance) {
|
|
1567
|
+
distances[j + 1] = distances[j];
|
|
1568
|
+
ords[j + 1] = ords[j];
|
|
1569
|
+
j -= 1;
|
|
1570
|
+
}
|
|
1571
|
+
distances[j + 1] = distance;
|
|
1572
|
+
ords[j + 1] = ord;
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
function siftUpGreatestFirst(ords, distances, start, ord, distance) {
|
|
1576
|
+
let index = start;
|
|
1577
|
+
while (index > 0) {
|
|
1578
|
+
const parent = index - 1 >> 1;
|
|
1579
|
+
if (!(distance > distances[parent])) break;
|
|
1580
|
+
ords[index] = ords[parent];
|
|
1581
|
+
distances[index] = distances[parent];
|
|
1582
|
+
index = parent;
|
|
1583
|
+
}
|
|
1584
|
+
ords[index] = ord;
|
|
1585
|
+
distances[index] = distance;
|
|
1586
|
+
}
|
|
1587
|
+
function siftUpLeastFirst(ords, distances, start, ord, distance) {
|
|
1588
|
+
let index = start;
|
|
1589
|
+
while (index > 0) {
|
|
1590
|
+
const parent = index - 1 >> 1;
|
|
1591
|
+
if (!(distance < distances[parent])) break;
|
|
1592
|
+
ords[index] = ords[parent];
|
|
1593
|
+
distances[index] = distances[parent];
|
|
1594
|
+
index = parent;
|
|
1595
|
+
}
|
|
1596
|
+
ords[index] = ord;
|
|
1597
|
+
distances[index] = distance;
|
|
1598
|
+
}
|
|
1599
|
+
function siftDownGreatestFirst(ords, distances, size, ord, distance) {
|
|
1600
|
+
let index = 0;
|
|
1601
|
+
for (; ; ) {
|
|
1602
|
+
const left = 2 * index + 1;
|
|
1603
|
+
const right = left + 1;
|
|
1604
|
+
let first = index;
|
|
1605
|
+
let firstDistance = distance;
|
|
1606
|
+
if (left < size && distances[left] > firstDistance) {
|
|
1607
|
+
first = left;
|
|
1608
|
+
firstDistance = distances[left];
|
|
1609
|
+
}
|
|
1610
|
+
if (right < size && distances[right] > firstDistance) first = right;
|
|
1611
|
+
if (first === index) break;
|
|
1612
|
+
ords[index] = ords[first];
|
|
1613
|
+
distances[index] = distances[first];
|
|
1614
|
+
index = first;
|
|
1615
|
+
}
|
|
1616
|
+
ords[index] = ord;
|
|
1617
|
+
distances[index] = distance;
|
|
1618
|
+
}
|
|
1619
|
+
function siftDownLeastFirst(ords, distances, size, ord, distance) {
|
|
1620
|
+
let index = 0;
|
|
1621
|
+
for (; ; ) {
|
|
1622
|
+
const left = 2 * index + 1;
|
|
1623
|
+
const right = left + 1;
|
|
1624
|
+
let first = index;
|
|
1625
|
+
let firstDistance = distance;
|
|
1626
|
+
if (left < size && distances[left] < firstDistance) {
|
|
1627
|
+
first = left;
|
|
1628
|
+
firstDistance = distances[left];
|
|
1629
|
+
}
|
|
1630
|
+
if (right < size && distances[right] < firstDistance) first = right;
|
|
1631
|
+
if (first === index) break;
|
|
1632
|
+
ords[index] = ords[first];
|
|
1633
|
+
distances[index] = distances[first];
|
|
1634
|
+
index = first;
|
|
1635
|
+
}
|
|
1636
|
+
ords[index] = ord;
|
|
1637
|
+
distances[index] = distance;
|
|
1638
|
+
}
|
|
1639
|
+
function pushHeap(heap, ord, distance) {
|
|
1640
|
+
ensureListCapacity(heap, heap.size + 1);
|
|
1641
|
+
const start = heap.size;
|
|
1642
|
+
heap.size += 1;
|
|
1643
|
+
if (heap.greatestFirst) siftUpGreatestFirst(heap.ords, heap.distances, start, ord, distance);
|
|
1644
|
+
else siftUpLeastFirst(heap.ords, heap.distances, start, ord, distance);
|
|
1645
|
+
}
|
|
1646
|
+
function popHeap(heap) {
|
|
1647
|
+
if (heap.size === 0) return false;
|
|
1648
|
+
heap.topOrd = heap.ords[0];
|
|
1649
|
+
heap.topDistance = heap.distances[0];
|
|
1650
|
+
heap.size -= 1;
|
|
1651
|
+
if (heap.size > 0) {
|
|
1652
|
+
const size = heap.size;
|
|
1653
|
+
const ord = heap.ords[size];
|
|
1654
|
+
const distance = heap.distances[size];
|
|
1655
|
+
if (heap.greatestFirst) siftDownGreatestFirst(heap.ords, heap.distances, size, ord, distance);
|
|
1656
|
+
else siftDownLeastFirst(heap.ords, heap.distances, size, ord, distance);
|
|
1657
|
+
}
|
|
1658
|
+
return true;
|
|
1659
|
+
}
|
|
1660
|
+
function resetHeap(heap) {
|
|
1661
|
+
heap.size = 0;
|
|
1662
|
+
}
|
|
1663
|
+
function drainHeapNearestFirst(heap, list) {
|
|
1664
|
+
const count = heap.size;
|
|
1665
|
+
ensureListCapacity(list, count);
|
|
1666
|
+
list.size = count;
|
|
1667
|
+
for (let i = count - 1; i >= 0; i--) {
|
|
1668
|
+
if (!popHeap(heap)) break;
|
|
1669
|
+
list.ords[i] = heap.topOrd;
|
|
1670
|
+
list.distances[i] = heap.topDistance;
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
function setSingleEntryPoint(workspace, ord) {
|
|
1674
|
+
workspace.entryPoints[0] = ord;
|
|
1675
|
+
workspace.entryPointCount = 1;
|
|
1676
|
+
}
|
|
1677
|
+
function setEntryPointsFromList(workspace, list) {
|
|
1678
|
+
if (list.size > workspace.entryPoints.length) {
|
|
1679
|
+
let capacity = workspace.entryPoints.length;
|
|
1680
|
+
while (capacity < list.size) capacity *= 2;
|
|
1681
|
+
workspace.entryPoints = new Int32Array(capacity);
|
|
1682
|
+
}
|
|
1683
|
+
workspace.entryPoints.set(list.ords.subarray(0, list.size));
|
|
1684
|
+
workspace.entryPointCount = list.size;
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
// src/vector/hnsw/last-links.ts
|
|
1688
|
+
function isLive(state, ord) {
|
|
1689
|
+
return nodeExists(state, ord) && !isTombstoned(state, ord);
|
|
1690
|
+
}
|
|
1691
|
+
function noteNodeLeftUnlinked(state) {
|
|
1692
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 1);
|
|
1693
|
+
}
|
|
1694
|
+
function listHolds(list, ord) {
|
|
1695
|
+
for (let i = 0; i < list.size; i++) if (list.ords[i] === ord) return true;
|
|
1696
|
+
return false;
|
|
1697
|
+
}
|
|
1698
|
+
function collectDropped(sorted, kept, dropped) {
|
|
1699
|
+
dropped.size = 0;
|
|
1700
|
+
let next = 0;
|
|
1701
|
+
for (let i = 0; i < sorted.size; i++) {
|
|
1702
|
+
const ord = sorted.ords[i];
|
|
1703
|
+
if (next < kept.size && kept.ords[next] === ord) {
|
|
1704
|
+
next += 1;
|
|
1705
|
+
continue;
|
|
1706
|
+
}
|
|
1707
|
+
if (listHolds(kept, ord)) continue;
|
|
1708
|
+
appendToList(dropped, ord, sorted.distances[i]);
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
function replaceFarthestShared(state, kept, held, ord, distance) {
|
|
1712
|
+
const tried = /* @__PURE__ */ new Set();
|
|
1713
|
+
for (; ; ) {
|
|
1714
|
+
let farthest = -1;
|
|
1715
|
+
for (let k = 0; k < kept.size; k++) {
|
|
1716
|
+
if (tried.has(k) || incomingCount(state.adjacency, kept.ords[k]) < 2) continue;
|
|
1717
|
+
if (farthest === -1 || kept.distances[k] > kept.distances[farthest]) farthest = k;
|
|
1718
|
+
}
|
|
1719
|
+
if (farthest === -1) return false;
|
|
1720
|
+
tried.add(farthest);
|
|
1721
|
+
const replaced = kept.ords[farthest];
|
|
1722
|
+
if (held(replaced) && !releaseUnlessLast(state.adjacency, replaced)) continue;
|
|
1723
|
+
kept.ords[farthest] = ord;
|
|
1724
|
+
kept.distances[farthest] = distance;
|
|
1725
|
+
return true;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
function writeKeepingLastLinks(state, ord, sorted, kept, held) {
|
|
1729
|
+
const adjacency = state.adjacency;
|
|
1730
|
+
const limit = state.Mmax0;
|
|
1731
|
+
const dropped = state.workspace.droppedLinks;
|
|
1732
|
+
collectDropped(sorted, kept, dropped);
|
|
1733
|
+
ensureListCapacity(kept, limit);
|
|
1734
|
+
for (let i = 0; i < dropped.size; i++) {
|
|
1735
|
+
const candidate = dropped.ords[i];
|
|
1736
|
+
if (!held(candidate)) continue;
|
|
1737
|
+
if (!isLive(state, candidate)) {
|
|
1738
|
+
noteUnlinked(adjacency, candidate);
|
|
1739
|
+
continue;
|
|
1740
|
+
}
|
|
1741
|
+
if (releaseUnlessLast(adjacency, candidate)) continue;
|
|
1742
|
+
if (kept.size < limit) {
|
|
1743
|
+
appendToList(kept, candidate, dropped.distances[i]);
|
|
1744
|
+
continue;
|
|
1745
|
+
}
|
|
1746
|
+
if (replaceFarthestShared(state, kept, held, candidate, dropped.distances[i])) continue;
|
|
1747
|
+
noteUnlinked(adjacency, candidate);
|
|
1748
|
+
noteNodeLeftUnlinked(state);
|
|
1749
|
+
}
|
|
1750
|
+
for (let k = 0; k < kept.size; k++) if (!held(kept.ords[k])) noteLinked(adjacency, kept.ords[k]);
|
|
1751
|
+
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
// src/vector/hnsw/connect-unreached.ts
|
|
1755
|
+
var NOT_REACHED_THROUGH_A_NODE = -1;
|
|
1756
|
+
function reach(walk, node, through) {
|
|
1757
|
+
walk.reached[node] = 1;
|
|
1758
|
+
walk.firstReachedThrough[node] = through;
|
|
1759
|
+
walk.queue[walk.tail++] = node;
|
|
1760
|
+
}
|
|
1761
|
+
function walkFrom(state, walk, start, through) {
|
|
1762
|
+
const adjacency = state.adjacency;
|
|
1763
|
+
if (walk.reached[start] === 1) return;
|
|
1764
|
+
reach(walk, start, through);
|
|
1765
|
+
while (walk.head < walk.tail) {
|
|
1766
|
+
const ord = walk.queue[walk.head++];
|
|
1767
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1768
|
+
if (base === ABSENT) continue;
|
|
1769
|
+
const count = adjacency.level0[base];
|
|
1770
|
+
for (let i = 1; i <= count; i++) {
|
|
1771
|
+
const next = adjacency.level0[base + i];
|
|
1772
|
+
if (next < 0 || next >= walk.reached.length || walk.reached[next] === 1 || !nodeExists(state, next)) continue;
|
|
1773
|
+
reach(walk, next, ord);
|
|
1774
|
+
}
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
function hasRoom(state, _walk, host) {
|
|
1778
|
+
const base = layerBase(state.adjacency, host, 0);
|
|
1779
|
+
return base !== ABSENT && state.adjacency.level0[base] < state.Mmax0;
|
|
1780
|
+
}
|
|
1781
|
+
function reachedThroughAnotherNode(walk, host, neighbour) {
|
|
1782
|
+
if (neighbour < 0 || neighbour >= walk.reached.length || walk.reached[neighbour] !== 1) return false;
|
|
1783
|
+
const through = walk.firstReachedThrough[neighbour];
|
|
1784
|
+
return through !== NOT_REACHED_THROUGH_A_NODE && through !== host;
|
|
1785
|
+
}
|
|
1786
|
+
function holdsLinkOutsideTheWalk(state, walk, host) {
|
|
1787
|
+
const adjacency = state.adjacency;
|
|
1788
|
+
const base = layerBase(adjacency, host, 0);
|
|
1789
|
+
if (base === ABSENT) return false;
|
|
1790
|
+
const count = adjacency.level0[base];
|
|
1791
|
+
for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
|
|
1792
|
+
return false;
|
|
1793
|
+
}
|
|
1794
|
+
function listOf(state, ord) {
|
|
1795
|
+
const adjacency = state.adjacency;
|
|
1796
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1797
|
+
if (base === ABSENT) return new Int32Array(0);
|
|
1798
|
+
return adjacency.level0.slice(base + 1, base + 1 + adjacency.level0[base]);
|
|
1799
|
+
}
|
|
1800
|
+
function nearestHost(state, walk, orphan, nearby, test) {
|
|
1801
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
1802
|
+
let best = -1;
|
|
1803
|
+
let bestDistance = Number.POSITIVE_INFINITY;
|
|
1804
|
+
const consider = (host) => {
|
|
1805
|
+
if (host === orphan || walk.reached[host] !== 1 || !isLive(state, host) || !test(state, walk, host)) return;
|
|
1806
|
+
const away = distance(orphan, host);
|
|
1807
|
+
if (best === -1 || away < bestDistance) {
|
|
1808
|
+
best = host;
|
|
1809
|
+
bestDistance = away;
|
|
1810
|
+
}
|
|
1811
|
+
};
|
|
1812
|
+
for (const host of nearby.list) consider(host);
|
|
1813
|
+
for (const host of listOf(state, orphan))
|
|
1814
|
+
if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
|
|
1815
|
+
if (best !== -1) return best;
|
|
1816
|
+
for (let host = 0; host < walk.reached.length; host++) consider(host);
|
|
1817
|
+
return best;
|
|
1818
|
+
}
|
|
1819
|
+
function takeOutFarthestLinkOutsideTheWalk(state, walk, host) {
|
|
1820
|
+
const adjacency = state.adjacency;
|
|
1821
|
+
const base = layerBase(adjacency, host, 0);
|
|
1822
|
+
if (base === ABSENT) return;
|
|
1823
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
1824
|
+
const count = adjacency.level0[base];
|
|
1825
|
+
let farthest = -1;
|
|
1826
|
+
let farthestDistance = Number.NEGATIVE_INFINITY;
|
|
1827
|
+
for (let i = 1; i <= count; i++) {
|
|
1828
|
+
const neighbour = adjacency.level0[base + i];
|
|
1829
|
+
if (!reachedThroughAnotherNode(walk, host, neighbour)) continue;
|
|
1830
|
+
const away = distance(host, neighbour);
|
|
1831
|
+
if (farthest === -1 || away > farthestDistance) {
|
|
1832
|
+
farthest = i;
|
|
1833
|
+
farthestDistance = away;
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
if (farthest === -1) return;
|
|
1837
|
+
const taken = adjacency.level0[base + farthest];
|
|
1838
|
+
for (let i = farthest; i < count; i++) adjacency.level0[base + i] = adjacency.level0[base + i + 1];
|
|
1839
|
+
adjacency.level0[base] = count - 1;
|
|
1840
|
+
noteUnlinked(adjacency, taken);
|
|
1841
|
+
}
|
|
1842
|
+
function appendTo(state, host, orphan) {
|
|
1843
|
+
const adjacency = state.adjacency;
|
|
1844
|
+
const base = layerBase(adjacency, host, 0);
|
|
1845
|
+
if (base === ABSENT) return;
|
|
1846
|
+
const count = adjacency.level0[base];
|
|
1847
|
+
adjacency.level0[base + count + 1] = orphan;
|
|
1848
|
+
adjacency.level0[base] = count + 1;
|
|
1849
|
+
noteLinked(adjacency, orphan);
|
|
1850
|
+
}
|
|
1851
|
+
function connectUnreached(state, nearby) {
|
|
1852
|
+
const slots = adjacencySlots(state.adjacency);
|
|
1853
|
+
const entry = entryPointOf(state);
|
|
1854
|
+
if (entry < 0 || entry >= slots || !nodeExists(state, entry)) return;
|
|
1855
|
+
const walk = {
|
|
1856
|
+
reached: new Uint8Array(slots),
|
|
1857
|
+
firstReachedThrough: new Int32Array(slots),
|
|
1858
|
+
queue: new Int32Array(slots),
|
|
1859
|
+
head: 0,
|
|
1860
|
+
tail: 0
|
|
1861
|
+
};
|
|
1862
|
+
walkFrom(state, walk, entry, NOT_REACHED_THROUGH_A_NODE);
|
|
1863
|
+
const held = { list: [], holds: new Uint8Array(slots) };
|
|
1864
|
+
for (let i = 0; i < nearby.length; i++) {
|
|
1865
|
+
const ord = nearby[i];
|
|
1866
|
+
if (ord < 0 || ord >= slots || held.holds[ord] === 1) continue;
|
|
1867
|
+
held.holds[ord] = 1;
|
|
1868
|
+
held.list.push(ord);
|
|
1869
|
+
}
|
|
1870
|
+
for (let orphan = 0; orphan < slots; orphan++) {
|
|
1871
|
+
if (walk.reached[orphan] === 1 || !isLive(state, orphan)) continue;
|
|
1872
|
+
let host = nearestHost(state, walk, orphan, held, hasRoom);
|
|
1873
|
+
if (host === -1) {
|
|
1874
|
+
host = nearestHost(state, walk, orphan, held, holdsLinkOutsideTheWalk);
|
|
1875
|
+
if (host === -1) continue;
|
|
1876
|
+
takeOutFarthestLinkOutsideTheWalk(state, walk, host);
|
|
1877
|
+
}
|
|
1878
|
+
appendTo(state, host, orphan);
|
|
1879
|
+
walkFrom(state, walk, orphan, host);
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
function holdsUnlinkedNode(state) {
|
|
1883
|
+
const slots = adjacencySlots(state.adjacency);
|
|
1884
|
+
const entry = entryPointOf(state);
|
|
1885
|
+
const incoming = state.adjacency.incoming;
|
|
1886
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
1887
|
+
if (incoming[ord] === 0 && ord !== entry && isLive(state, ord)) return true;
|
|
1888
|
+
}
|
|
1889
|
+
return false;
|
|
1890
|
+
}
|
|
1891
|
+
function connectThroughEitherCore(state, nearby) {
|
|
1892
|
+
const field = nativeFieldFor();
|
|
1893
|
+
if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
|
|
1894
|
+
connectUnreached(state, nearby);
|
|
1895
|
+
}
|
|
1896
|
+
function connectAfterRemoval(state, nearby) {
|
|
1897
|
+
connectThroughEitherCore(state, nearby);
|
|
1898
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1899
|
+
}
|
|
1900
|
+
function connectAfterBuild(state) {
|
|
1901
|
+
connectThroughEitherCore(state, new Int32Array(0));
|
|
1902
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1903
|
+
}
|
|
1904
|
+
function connectAfterPlacement(state, holdsGraphLock) {
|
|
1905
|
+
if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
|
|
1906
|
+
if (!holdsGraphLock) lockGraphExclusive(state.locks);
|
|
1907
|
+
try {
|
|
1908
|
+
if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
|
|
1909
|
+
connectThroughEitherCore(state, new Int32Array(0));
|
|
1910
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1911
|
+
} finally {
|
|
1912
|
+
if (!holdsGraphLock) unlockGraphExclusive(state.locks);
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
function baseNeighboursOfRemoved(state, removed) {
|
|
1916
|
+
const leaving = new Set(removed);
|
|
1917
|
+
const nearby = [];
|
|
1918
|
+
for (const ord of removed) {
|
|
1919
|
+
for (const neighbour of listOf(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
|
|
1920
|
+
}
|
|
1921
|
+
return Int32Array.from(nearby);
|
|
1922
|
+
}
|
|
1923
|
+
function leaveIncomingUncounted(state) {
|
|
1924
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1925
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 1);
|
|
1926
|
+
}
|
|
1927
|
+
function countIncomingBeforeWriting(state, holdsGraphLock) {
|
|
1928
|
+
if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
|
|
1929
|
+
if (!holdsGraphLock) lockGraphExclusive(state.locks);
|
|
1930
|
+
try {
|
|
1931
|
+
if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
|
|
1932
|
+
const adjacency = state.adjacency;
|
|
1933
|
+
const slots = adjacencySlots(adjacency);
|
|
1934
|
+
if (slots > 0) reachNode(adjacency, slots - 1);
|
|
1935
|
+
adjacency.incoming.fill(0);
|
|
1936
|
+
countEveryLink(adjacency, slots);
|
|
1937
|
+
if (holdsUnlinkedNode(state)) connectThroughEitherCore(state, new Int32Array(0));
|
|
1938
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1939
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
|
|
1940
|
+
} finally {
|
|
1941
|
+
if (!holdsGraphLock) unlockGraphExclusive(state.locks);
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
// src/vector/hnsw/graph-ops.ts
|
|
1946
|
+
function readNeighborsLocked(state, ord, layer) {
|
|
1947
|
+
for (; ; ) {
|
|
1948
|
+
const version = beginNodeRead(state.locks, ord);
|
|
1949
|
+
const count = readNeighbors(state.adjacency, ord, layer, state.neighborScratch);
|
|
1950
|
+
if (nodeReadHeld(state.locks, ord, version)) return count;
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
function searchLayer(state, qVec, qMag, ef, layer, metric, excludeTombstones, distFn, results, accepted) {
|
|
1954
|
+
const getDistance = distFn ?? ((ord) => queryDistanceByOrd(state, qVec, qMag, ord, metric));
|
|
1955
|
+
const workspace = state.workspace;
|
|
1956
|
+
const frontier = workspace.frontier;
|
|
1957
|
+
const found = workspace.found;
|
|
1958
|
+
const scratch = state.neighborScratch;
|
|
1959
|
+
const stamp = nextVisitStamp(state);
|
|
1960
|
+
resetHeap(frontier);
|
|
1961
|
+
resetHeap(found);
|
|
1962
|
+
let furthestDist = Number.POSITIVE_INFINITY;
|
|
1963
|
+
for (let i = 0; i < workspace.entryPointCount; i++) {
|
|
1964
|
+
const epOrd = workspace.entryPoints[i];
|
|
1965
|
+
ensureVisited(state, epOrd + 1);
|
|
1966
|
+
if (state.visited[epOrd] === stamp) continue;
|
|
1967
|
+
state.visited[epOrd] = stamp;
|
|
1968
|
+
if (!nodeExists(state, epOrd)) continue;
|
|
1969
|
+
const dist = getDistance(epOrd);
|
|
1970
|
+
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
1971
|
+
pushHeap(frontier, epOrd, dist);
|
|
1972
|
+
if (excludeTombstones && isTombstoned(state, epOrd)) continue;
|
|
1973
|
+
if (accepted && !ordinalFilterHas(accepted, epOrd)) continue;
|
|
1974
|
+
pushHeap(found, epOrd, dist);
|
|
1975
|
+
if (found.size > ef) {
|
|
1976
|
+
popHeap(found);
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
if (found.size > 0) furthestDist = found.distances[0];
|
|
1980
|
+
while (popHeap(frontier)) {
|
|
1981
|
+
if (frontier.topDistance > furthestDist && found.size >= ef) break;
|
|
1982
|
+
const count = readNeighborsLocked(state, frontier.topOrd, layer);
|
|
1983
|
+
for (let i = 0; i < count; i++) {
|
|
1984
|
+
const neighborOrd = scratch[i];
|
|
1985
|
+
ensureVisited(state, neighborOrd + 1);
|
|
1986
|
+
if (state.visited[neighborOrd] === stamp) continue;
|
|
1987
|
+
state.visited[neighborOrd] = stamp;
|
|
1988
|
+
if (!nodeExists(state, neighborOrd)) continue;
|
|
1989
|
+
const dist = getDistance(neighborOrd);
|
|
1990
|
+
if (dist === Number.POSITIVE_INFINITY) continue;
|
|
1991
|
+
if (dist < furthestDist || found.size < ef) {
|
|
1992
|
+
pushHeap(frontier, neighborOrd, dist);
|
|
1993
|
+
if (excludeTombstones && isTombstoned(state, neighborOrd)) continue;
|
|
1994
|
+
if (accepted && !ordinalFilterHas(accepted, neighborOrd)) continue;
|
|
1995
|
+
pushHeap(found, neighborOrd, dist);
|
|
1996
|
+
if (found.size > ef) {
|
|
1997
|
+
popHeap(found);
|
|
1998
|
+
}
|
|
1999
|
+
furthestDist = found.distances[0];
|
|
2000
|
+
}
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
drainHeapNearestFirst(found, results);
|
|
2004
|
+
}
|
|
2005
|
+
function selectNeighborsHeuristic(state, candidates, maxConnections, distance, selected) {
|
|
2006
|
+
const working = state.workspace.working;
|
|
2007
|
+
copyList(candidates, working);
|
|
2008
|
+
sortListByDistance(working);
|
|
2009
|
+
ensureListCapacity(selected, Math.min(maxConnections, working.size));
|
|
2010
|
+
selected.size = 0;
|
|
2011
|
+
for (let i = 0; i < working.size; i++) {
|
|
2012
|
+
if (selected.size >= maxConnections) break;
|
|
2013
|
+
const candidateOrd = working.ords[i];
|
|
2014
|
+
const candidateDistance = working.distances[i];
|
|
2015
|
+
let accepted = true;
|
|
2016
|
+
for (let s = 0; s < selected.size; s++) {
|
|
2017
|
+
const distBetween = distance(candidateOrd, selected.ords[s]);
|
|
2018
|
+
if (candidateDistance >= distBetween) {
|
|
2019
|
+
accepted = false;
|
|
2020
|
+
break;
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
if (!accepted) continue;
|
|
2024
|
+
selected.ords[selected.size] = candidateOrd;
|
|
2025
|
+
selected.distances[selected.size] = candidateDistance;
|
|
2026
|
+
selected.size += 1;
|
|
2027
|
+
}
|
|
2028
|
+
keepFarthestRejected(working, maxConnections, selected);
|
|
2029
|
+
}
|
|
2030
|
+
function holdsOrd(list, ord) {
|
|
2031
|
+
for (let i = 0; i < list.size; i++) if (list.ords[i] === ord) return true;
|
|
2032
|
+
return false;
|
|
2033
|
+
}
|
|
2034
|
+
function keepFarthestRejected(sorted, maxConnections, selected) {
|
|
2035
|
+
if (selected.size * 2 >= maxConnections) return;
|
|
2036
|
+
let kept = 0;
|
|
2037
|
+
for (let i = sorted.size - 1; i >= 0 && kept < FARTHEST_REJECTED_LINKS_KEPT; i--) {
|
|
2038
|
+
if (selected.size >= maxConnections) return;
|
|
2039
|
+
if (holdsOrd(selected, sorted.ords[i])) continue;
|
|
2040
|
+
selected.ords[selected.size] = sorted.ords[i];
|
|
2041
|
+
selected.distances[selected.size] = sorted.distances[i];
|
|
2042
|
+
selected.size += 1;
|
|
2043
|
+
kept += 1;
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
function pruneConnections(state, ord, layer, distance) {
|
|
2047
|
+
const adjacency = state.adjacency;
|
|
2048
|
+
const base = layerBase(adjacency, ord, layer);
|
|
2049
|
+
if (base === -1) return;
|
|
2050
|
+
const mc = maxConns(state, layer);
|
|
2051
|
+
const neighbors = layerArray(adjacency, layer);
|
|
2052
|
+
const count = neighbors[base];
|
|
2053
|
+
if (count <= mc) return;
|
|
2054
|
+
const candidates = state.workspace.pruneCandidates;
|
|
2055
|
+
candidates.size = 0;
|
|
2056
|
+
ensureListCapacity(candidates, count);
|
|
2057
|
+
for (let i = 1; i <= count; i++) {
|
|
2058
|
+
const connOrd = neighbors[base + i];
|
|
2059
|
+
const dist = distance(ord, connOrd);
|
|
2060
|
+
if (dist === Number.POSITIVE_INFINITY) {
|
|
2061
|
+
if (layer === 0) noteUnlinked(adjacency, connOrd);
|
|
2062
|
+
continue;
|
|
2063
|
+
}
|
|
2064
|
+
candidates.ords[candidates.size] = connOrd;
|
|
2065
|
+
candidates.distances[candidates.size] = dist;
|
|
2066
|
+
candidates.size += 1;
|
|
2067
|
+
}
|
|
2068
|
+
const kept = state.workspace.pruneSelection;
|
|
2069
|
+
selectNeighborsHeuristic(state, candidates, mc, distance, kept);
|
|
2070
|
+
if (layer === 0) {
|
|
2071
|
+
writeKeepingLastLinks(state, ord, state.workspace.working, kept, everyCandidateWasHeld);
|
|
2072
|
+
return;
|
|
2073
|
+
}
|
|
2074
|
+
writeNeighbors(adjacency, ord, layer, kept.ords, kept.size);
|
|
2075
|
+
}
|
|
2076
|
+
function everyCandidateWasHeld() {
|
|
2077
|
+
return true;
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
// src/vector/hnsw/insertion-order.ts
|
|
2081
|
+
function shuffleInPlace(items) {
|
|
2082
|
+
for (let i = items.length - 1; i > 0; i--) {
|
|
2083
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
2084
|
+
const held = items[i];
|
|
2085
|
+
items[i] = items[j];
|
|
2086
|
+
items[j] = held;
|
|
2087
|
+
}
|
|
2088
|
+
return items;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
// src/vector/hnsw/mutation.ts
|
|
2092
|
+
function clearTombstone(state, ord) {
|
|
2093
|
+
if (!reachTombstone(state, ord) || Atomics.load(state.tombstones, ord) !== 1) return;
|
|
2094
|
+
Atomics.sub(state.header, GRAPH_TOMBSTONE_COUNT, 1);
|
|
2095
|
+
Atomics.store(state.tombstones, ord, 0);
|
|
2096
|
+
}
|
|
2097
|
+
function highestNode(state, includeTombstoned) {
|
|
2098
|
+
let bestOrd = -1;
|
|
2099
|
+
let bestLayer = -1;
|
|
2100
|
+
const slots = adjacencySlots(state.adjacency);
|
|
2101
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
2102
|
+
const level2 = nodeMaxLayer(state, ord);
|
|
2103
|
+
if (level2 === -1) continue;
|
|
2104
|
+
if (!includeTombstoned && isTombstoned(state, ord)) continue;
|
|
2105
|
+
if (level2 > bestLayer) {
|
|
2106
|
+
bestLayer = level2;
|
|
2107
|
+
bestOrd = ord;
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
return bestOrd;
|
|
2111
|
+
}
|
|
2112
|
+
function setEntryPoint(state, ord) {
|
|
2113
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, ord);
|
|
2114
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, ord === -1 ? -1 : nodeMaxLayer(state, ord));
|
|
2115
|
+
}
|
|
2116
|
+
function claimEntryIfEmpty(state, ord, level2) {
|
|
2117
|
+
lockEntry(state.locks);
|
|
2118
|
+
try {
|
|
2119
|
+
if (Atomics.load(state.header, GRAPH_ENTRY_POINT) !== -1) return false;
|
|
2120
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, ord);
|
|
2121
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, level2);
|
|
2122
|
+
return true;
|
|
2123
|
+
} finally {
|
|
2124
|
+
unlockEntry(state.locks);
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
function raiseEntry(state, ord, level2) {
|
|
2128
|
+
lockEntry(state.locks);
|
|
2129
|
+
try {
|
|
2130
|
+
if (level2 <= Atomics.load(state.header, GRAPH_TOP_LAYER)) return;
|
|
2131
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, ord);
|
|
2132
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, level2);
|
|
2133
|
+
} finally {
|
|
2134
|
+
unlockEntry(state.locks);
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
function placementDistance(state, vector, fullPrecision) {
|
|
2138
|
+
const quantizer = state.quantizer;
|
|
2139
|
+
if (quantizer === void 0 || !buildsFromCodes(state)) return fullPrecision;
|
|
2140
|
+
if (state.store.handles.vectorFiles.length > 0) return quantizer.placementDistance(vector) ?? fullPrecision;
|
|
2141
|
+
const prepared = quantizer.prepareQuery(vector);
|
|
2142
|
+
return prepared === null ? fullPrecision : quantizer.preparedDistance(prepared);
|
|
2143
|
+
}
|
|
2144
|
+
function selectNeighborsPerLayer(state, ord, level2) {
|
|
2145
|
+
const metric = state.buildMetric;
|
|
2146
|
+
const workspace = state.workspace;
|
|
2147
|
+
const candidates = workspace.traversal;
|
|
2148
|
+
const entry = state.store.entryForOrdinal(ord);
|
|
2149
|
+
if (entry === void 0) return -1;
|
|
2150
|
+
const fullPrecision = state.store.ordinalDistance(ord, metric);
|
|
2151
|
+
const insertDistFn = placementDistance(state, entry.vector, fullPrecision);
|
|
2152
|
+
const topLayer = topLayerOf(state);
|
|
2153
|
+
setSingleEntryPoint(workspace, entryPointOf(state));
|
|
2154
|
+
for (let layer = topLayer; layer > level2; layer--) {
|
|
2155
|
+
searchLayer(state, entry.vector, entry.magnitude, 1, layer, metric, false, insertDistFn, candidates);
|
|
2156
|
+
if (candidates.size > 0) setSingleEntryPoint(workspace, candidates.ords[0]);
|
|
2157
|
+
}
|
|
2158
|
+
const linkTop = Math.min(level2, topLayer);
|
|
2159
|
+
const selections = linkSelectionsFor(workspace, linkTop + 1);
|
|
2160
|
+
const distance = nodeDistanceFunction(state, metric);
|
|
2161
|
+
for (let layer = linkTop; layer >= 0; layer--) {
|
|
2162
|
+
searchLayer(state, entry.vector, entry.magnitude, state.efCons, layer, metric, false, insertDistFn, candidates);
|
|
2163
|
+
selectNeighborsHeuristic(state, candidates, maxConns(state, layer), distance, selections[layer]);
|
|
2164
|
+
if (candidates.size > 0) setEntryPointsFromList(workspace, candidates);
|
|
2165
|
+
}
|
|
2166
|
+
return linkTop;
|
|
2167
|
+
}
|
|
2168
|
+
function writeOwnLists(state, ord, linkTop) {
|
|
2169
|
+
const selections = state.workspace.linkSelections;
|
|
2170
|
+
lockNodeWrite(state.locks, ord);
|
|
2171
|
+
try {
|
|
2172
|
+
for (let layer = linkTop; layer >= 0; layer--) {
|
|
2173
|
+
const selected = selections[layer];
|
|
2174
|
+
replaceNeighbors(state.adjacency, ord, layer, selected.ords, selected.size);
|
|
2175
|
+
}
|
|
2176
|
+
} finally {
|
|
2177
|
+
unlockNodeWrite(state.locks, ord);
|
|
2178
|
+
}
|
|
2179
|
+
}
|
|
2180
|
+
function linkNeighborsBack(state, ord, linkTop) {
|
|
2181
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
2182
|
+
const selections = state.workspace.linkSelections;
|
|
2183
|
+
for (let layer = linkTop; layer >= 0; layer--) {
|
|
2184
|
+
const selected = selections[layer];
|
|
2185
|
+
for (let i = 0; i < selected.size; i++) {
|
|
2186
|
+
const neighborOrd = selected.ords[i];
|
|
2187
|
+
if (layer > nodeMaxLayer(state, neighborOrd)) continue;
|
|
2188
|
+
lockNodeWrite(state.locks, neighborOrd);
|
|
2189
|
+
try {
|
|
2190
|
+
addNeighbor(state.adjacency, neighborOrd, layer, ord);
|
|
2191
|
+
pruneConnections(state, neighborOrd, layer, distance);
|
|
2192
|
+
} finally {
|
|
2193
|
+
unlockNodeWrite(state.locks, neighborOrd);
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
function linkNode(state, ord, level2) {
|
|
2199
|
+
const topLayer = topLayerOf(state);
|
|
2200
|
+
const formerEntry = entryPointOf(state);
|
|
2201
|
+
const linkTop = selectNeighborsPerLayer(state, ord, level2);
|
|
2202
|
+
if (linkTop < 0) return;
|
|
2203
|
+
writeOwnLists(state, ord, linkTop);
|
|
2204
|
+
linkNeighborsBack(state, ord, linkTop);
|
|
2205
|
+
if (level2 <= topLayer) return;
|
|
2206
|
+
raiseEntry(state, ord, level2);
|
|
2207
|
+
const demoted = entryPointOf(state) !== formerEntry && isLive(state, formerEntry);
|
|
2208
|
+
if (demoted && incomingCount(state.adjacency, formerEntry) === 0) noteNodeLeftUnlinked(state);
|
|
2209
|
+
}
|
|
2210
|
+
function writeRecordBeforePlacement(state, ord) {
|
|
2211
|
+
const quantizer = state.quantizer;
|
|
2212
|
+
if (quantizer === void 0 || !buildsFromCodes(state)) return;
|
|
2213
|
+
quantizer.writeCodes(ord);
|
|
2214
|
+
}
|
|
2215
|
+
function placeThroughTheCore(state, ord, holdsGraphLock) {
|
|
2216
|
+
let field = nativeFieldFor();
|
|
2217
|
+
if (field === null) return false;
|
|
2218
|
+
const level2 = randomLevel(state.mL);
|
|
2219
|
+
for (let request = 0; request < PLACEMENT_ROOM_REQUESTS && field !== null; request++) {
|
|
2220
|
+
ensureUpperRoom(state.adjacency, level2);
|
|
2221
|
+
const placement = nativePlace(state, field, state.buildMetric, ord, level2, holdsGraphLock);
|
|
2222
|
+
if (placement === "placed") return true;
|
|
2223
|
+
if (placement === "failed") return false;
|
|
2224
|
+
ensureCapacity(state, ord + 1);
|
|
2225
|
+
publishGraphCapacity(state.adjacency.handles);
|
|
2226
|
+
field = nativeFieldFor();
|
|
2227
|
+
}
|
|
2228
|
+
return false;
|
|
2229
|
+
}
|
|
2230
|
+
function insertNode(state, ord, holdsGraphLock = false) {
|
|
2231
|
+
if (!state.store.holdsOrdinal(ord)) return false;
|
|
2232
|
+
ensureCapacity(state, ord + 1);
|
|
2233
|
+
if (nodeExists(state, ord)) return false;
|
|
2234
|
+
countIncomingBeforeWriting(state, holdsGraphLock);
|
|
2235
|
+
writeRecordBeforePlacement(state, ord);
|
|
2236
|
+
if (placeThroughTheCore(state, ord, holdsGraphLock)) {
|
|
2237
|
+
connectAfterPlacement(state, holdsGraphLock);
|
|
2238
|
+
return true;
|
|
2239
|
+
}
|
|
2240
|
+
if (nodeExists(state, ord)) return false;
|
|
2241
|
+
if (!holdsGraphLock) lockGraphShared(state.locks);
|
|
2242
|
+
try {
|
|
2243
|
+
clearTombstone(state, ord);
|
|
2244
|
+
const level2 = randomLevel(state.mL);
|
|
2245
|
+
createNode(state.adjacency, ord, level2);
|
|
2246
|
+
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
2247
|
+
if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
|
|
2248
|
+
} finally {
|
|
2249
|
+
if (!holdsGraphLock) unlockGraphShared(state.locks);
|
|
2250
|
+
}
|
|
2251
|
+
connectAfterPlacement(state, holdsGraphLock);
|
|
2252
|
+
return true;
|
|
2253
|
+
}
|
|
2254
|
+
function removeNodeEager(state, ord, excludeOrds) {
|
|
2255
|
+
const maxLayer = nodeMaxLayer(state, ord);
|
|
2256
|
+
if (maxLayer === -1) return;
|
|
2257
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
2258
|
+
for (let layer = 0; layer <= maxLayer; layer++) {
|
|
2259
|
+
const formerNeighbors = collectNeighbors(state.adjacency, ord, layer);
|
|
2260
|
+
for (const neighborOrd of formerNeighbors) {
|
|
2261
|
+
if (layer <= nodeMaxLayer(state, neighborOrd)) {
|
|
2262
|
+
removeNeighbor(state.adjacency, neighborOrd, layer, ord);
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
for (const neighborOrd of formerNeighbors) {
|
|
2266
|
+
if (layer > nodeMaxLayer(state, neighborOrd)) continue;
|
|
2267
|
+
const mc = maxConns(state, layer);
|
|
2268
|
+
if (neighborCount(state.adjacency, neighborOrd, layer) >= mc) continue;
|
|
2269
|
+
const heldNow = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
|
|
2270
|
+
const candidateOrds = new Set(heldNow);
|
|
2271
|
+
for (const otherOrd of formerNeighbors) {
|
|
2272
|
+
if (otherOrd !== neighborOrd && otherOrd !== ord) {
|
|
2273
|
+
if (excludeOrds?.has(otherOrd)) continue;
|
|
2274
|
+
candidateOrds.add(otherOrd);
|
|
2275
|
+
}
|
|
2276
|
+
}
|
|
2277
|
+
const candidates = state.workspace.repairCandidates;
|
|
2278
|
+
candidates.size = 0;
|
|
2279
|
+
for (const candOrd of candidateOrds) {
|
|
2280
|
+
const dist = distance(neighborOrd, candOrd);
|
|
2281
|
+
if (dist === Number.POSITIVE_INFINITY) {
|
|
2282
|
+
if (layer === 0 && heldNow.has(candOrd)) noteUnlinked(state.adjacency, candOrd);
|
|
2283
|
+
continue;
|
|
2284
|
+
}
|
|
2285
|
+
appendToList(candidates, candOrd, dist);
|
|
2286
|
+
}
|
|
2287
|
+
const selected = state.workspace.repairSelection;
|
|
2288
|
+
selectNeighborsHeuristic(state, candidates, mc, distance, selected);
|
|
2289
|
+
if (layer === 0) {
|
|
2290
|
+
writeKeepingLastLinks(state, neighborOrd, state.workspace.working, selected, (held) => heldNow.has(held));
|
|
2291
|
+
} else {
|
|
2292
|
+
replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
|
|
2293
|
+
}
|
|
2294
|
+
for (let i = 0; i < selected.size; i++) {
|
|
2295
|
+
const newConnOrd = selected.ords[i];
|
|
2296
|
+
if (layer <= nodeMaxLayer(state, newConnOrd)) {
|
|
2297
|
+
addNeighbor(state.adjacency, newConnOrd, layer, neighborOrd);
|
|
2298
|
+
pruneConnections(state, newConnOrd, layer, distance);
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
const formerBaseNeighbors = collectNeighbors(state.adjacency, ord, 0);
|
|
2304
|
+
for (const neighborOrd of formerBaseNeighbors) noteUnlinked(state.adjacency, neighborOrd);
|
|
2305
|
+
deleteNode(state.adjacency, ord);
|
|
2306
|
+
Atomics.sub(state.header, GRAPH_NODE_COUNT, 1);
|
|
2307
|
+
clearTombstone(state, ord);
|
|
2308
|
+
if (entryPointOf(state) === ord) {
|
|
2309
|
+
if (Atomics.load(state.header, GRAPH_NODE_COUNT) === 0) {
|
|
2310
|
+
setEntryPoint(state, -1);
|
|
2311
|
+
return;
|
|
2312
|
+
}
|
|
2313
|
+
const live = highestNode(state, false);
|
|
2314
|
+
setEntryPoint(state, live === -1 ? highestNode(state, true) : live);
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
function markTombstone(state, ord) {
|
|
2318
|
+
if (!nodeExists(state, ord) || !reachTombstone(state, ord)) return;
|
|
2319
|
+
const field = nativeFieldFor();
|
|
2320
|
+
if (field !== null && nativeRemove(state, field, ord)) return;
|
|
2321
|
+
if (Atomics.compareExchange(state.tombstones, ord, 0, 1) === 0) {
|
|
2322
|
+
Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
function compactTombstones(state) {
|
|
2326
|
+
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
2327
|
+
countIncomingBeforeWriting(state, true);
|
|
2328
|
+
const tombstonedOrds = [];
|
|
2329
|
+
const slots = adjacencySlots(state.adjacency);
|
|
2330
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
2331
|
+
if (isTombstoned(state, ord) && nodeExists(state, ord)) {
|
|
2332
|
+
tombstonedOrds.push(ord);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
const nearby = baseNeighboursOfRemoved(state, tombstonedOrds);
|
|
2336
|
+
const field = nativeFieldFor();
|
|
2337
|
+
if (field === null || !nativeCompact(state, field, state.buildMetric)) {
|
|
2338
|
+
const excl = new Set(tombstonedOrds);
|
|
2339
|
+
for (const ord of tombstonedOrds) {
|
|
2340
|
+
removeNodeEager(state, ord, excl);
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
connectAfterRemoval(state, nearby);
|
|
2344
|
+
}
|
|
2345
|
+
function resetGraph(state) {
|
|
2346
|
+
resetAdjacency(state.adjacency);
|
|
2347
|
+
state.tombstones = fixedView(state.adjacency.handles.tombstones, Uint8Array);
|
|
2348
|
+
state.tombstones.fill(0);
|
|
2349
|
+
Atomics.store(state.header, GRAPH_TOMBSTONE_COUNT, 0);
|
|
2350
|
+
Atomics.store(state.header, GRAPH_NODE_COUNT, 0);
|
|
2351
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, -1);
|
|
2352
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, -1);
|
|
2353
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
|
|
2354
|
+
}
|
|
2355
|
+
function rebuild(state) {
|
|
2356
|
+
const nodeCount = Atomics.load(state.header, GRAPH_NODE_COUNT);
|
|
2357
|
+
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0 && nodeCount === 0) return;
|
|
2358
|
+
const liveOrds = [];
|
|
2359
|
+
const slots = adjacencySlots(state.adjacency);
|
|
2360
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
2361
|
+
if (nodeExists(state, ord) && !isTombstoned(state, ord)) {
|
|
2362
|
+
liveOrds.push(ord);
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
resetGraph(state);
|
|
2366
|
+
for (const ord of shuffleInPlace(liveOrds)) {
|
|
2367
|
+
if (!state.store.holdsOrdinal(ord)) continue;
|
|
2368
|
+
insertNode(state, ord, true);
|
|
2369
|
+
}
|
|
2370
|
+
connectAfterBuild(state);
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2373
|
+
// src/vector/native/walk.ts
|
|
2374
|
+
function copyCandidates(store, count, target) {
|
|
2375
|
+
ensureListCapacity(target, count);
|
|
2376
|
+
target.ords.set(store.ordinals.subarray(0, count));
|
|
2377
|
+
target.distances.set(store.distances.subarray(0, count));
|
|
2378
|
+
target.size = count;
|
|
2379
|
+
}
|
|
2380
|
+
function nativeTraverse(state, field, query, candidateCount, metric, filter, candidates) {
|
|
2381
|
+
const store = field.store;
|
|
2382
|
+
reserveOrdinals(store, candidateCount);
|
|
2383
|
+
let count;
|
|
2384
|
+
try {
|
|
2385
|
+
count = field.core.search(
|
|
2386
|
+
field.graph,
|
|
2387
|
+
store.handle,
|
|
2388
|
+
query,
|
|
2389
|
+
nativeMetricCode(metric),
|
|
2390
|
+
candidateCount,
|
|
2391
|
+
state.locks.threadSlot,
|
|
2392
|
+
store.ordinals,
|
|
2393
|
+
store.distances,
|
|
2394
|
+
filter === void 0 ? null : filter.bits
|
|
2395
|
+
);
|
|
2396
|
+
} catch (error) {
|
|
2397
|
+
stopUsingTheNativeSearchCore(error);
|
|
2398
|
+
return false;
|
|
2399
|
+
}
|
|
2400
|
+
if (count < 0) return false;
|
|
2401
|
+
copyCandidates(store, count, candidates);
|
|
2402
|
+
return true;
|
|
2403
|
+
}
|
|
2404
|
+
function nativeRescoredHits(store, candidates, query, depth, metric, minSimilarity, filter) {
|
|
2405
|
+
const nearest = Math.min(candidates.size, depth);
|
|
2406
|
+
reserveOrdinals(store, nearest);
|
|
2407
|
+
const ordinals = store.ordinals;
|
|
2408
|
+
let kept = 0;
|
|
2409
|
+
for (let i = 0; i < candidates.size && kept < nearest; i++) {
|
|
2410
|
+
const ord = candidates.ords[i];
|
|
2411
|
+
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
2412
|
+
ordinals[kept++] = ord;
|
|
2413
|
+
}
|
|
2414
|
+
const distances = nativeScores(store, query, metric, kept);
|
|
2415
|
+
if (distances === null) return null;
|
|
2416
|
+
const hits = [];
|
|
2417
|
+
for (let i = 0; i < kept; i++) {
|
|
2418
|
+
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
2419
|
+
const score = toScore(distances[i], metric);
|
|
2420
|
+
if (score < minSimilarity) continue;
|
|
2421
|
+
hits.push({ ord: ordinals[i], score });
|
|
2422
|
+
}
|
|
2423
|
+
return hits;
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
// src/vector/search-depth/constants.ts
|
|
2427
|
+
var NARROW_DIMENSION_LIMIT = 768;
|
|
2428
|
+
var WIDE_DIMENSION_LIMIT = 1024;
|
|
2429
|
+
var DEFAULT_EF_SEARCH_MARGIN_BY_BAND = {
|
|
2430
|
+
narrow: 50,
|
|
2431
|
+
middle: 75,
|
|
2432
|
+
wide: 0
|
|
2433
|
+
};
|
|
2434
|
+
var DEFAULT_RESCORE_DEPTH_FLOOR_PER_LIMIT_BY_BITS = {
|
|
2435
|
+
1: 3,
|
|
2436
|
+
2: 3,
|
|
2437
|
+
4: 1,
|
|
2438
|
+
8: 1
|
|
2439
|
+
};
|
|
2440
|
+
var DEFAULT_RESCORE_DEPTH_BY_BITS = {
|
|
2441
|
+
1: {
|
|
2442
|
+
narrow: { perLimit: 3.23, perSquareRootOfLimit: 55.9 },
|
|
2443
|
+
middle: { perLimit: 2.2, perSquareRootOfLimit: 27 },
|
|
2444
|
+
wide: { perLimit: 3, perSquareRootOfLimit: 0 }
|
|
2445
|
+
},
|
|
2446
|
+
2: {
|
|
2447
|
+
narrow: { perLimit: 2.6, perSquareRootOfLimit: 22.5 },
|
|
2448
|
+
middle: { perLimit: 1, perSquareRootOfLimit: 16 },
|
|
2449
|
+
wide: { perLimit: 3, perSquareRootOfLimit: 0 }
|
|
2450
|
+
},
|
|
2451
|
+
4: {
|
|
2452
|
+
narrow: { perLimit: 5, perSquareRootOfLimit: 0 },
|
|
2453
|
+
middle: { perLimit: 5, perSquareRootOfLimit: 0 },
|
|
2454
|
+
wide: { perLimit: 2, perSquareRootOfLimit: 0 }
|
|
2455
|
+
},
|
|
2456
|
+
8: {
|
|
2457
|
+
narrow: { perLimit: 2, perSquareRootOfLimit: 0 },
|
|
2458
|
+
middle: { perLimit: 2, perSquareRootOfLimit: 0 },
|
|
2459
|
+
wide: { perLimit: 2, perSquareRootOfLimit: 0 }
|
|
2460
|
+
}
|
|
2461
|
+
};
|
|
2462
|
+
|
|
2463
|
+
// src/vector/search-depth/index.ts
|
|
2464
|
+
function dimensionBand(dimension) {
|
|
2465
|
+
if (dimension < NARROW_DIMENSION_LIMIT) return "narrow";
|
|
2466
|
+
return dimension < WIDE_DIMENSION_LIMIT ? "middle" : "wide";
|
|
2467
|
+
}
|
|
2468
|
+
function defaultRescoreDepth(bits, dimension, k) {
|
|
2469
|
+
const { perLimit, perSquareRootOfLimit } = DEFAULT_RESCORE_DEPTH_BY_BITS[bits][dimensionBand(dimension)];
|
|
2470
|
+
const floor = DEFAULT_RESCORE_DEPTH_FLOOR_PER_LIMIT_BY_BITS[bits] * k;
|
|
2471
|
+
return Math.max(floor, Math.ceil(perLimit * k + perSquareRootOfLimit * Math.sqrt(k)));
|
|
2472
|
+
}
|
|
2473
|
+
function defaultEfSearch(dimension, k) {
|
|
2474
|
+
return Math.max(DEFAULT_EF_SEARCH, k + DEFAULT_EF_SEARCH_MARGIN_BY_BAND[dimensionBand(dimension)]);
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
// src/vector/hnsw/search.ts
|
|
2478
|
+
function traverse(state, query, k, searchMetric, options) {
|
|
2479
|
+
const calibrated = activeQuantizer(state);
|
|
2480
|
+
const quantizer = calibrated === void 0 || calibrated.size === 0 ? void 0 : calibrated;
|
|
2481
|
+
const useQuantized = quantizer !== void 0;
|
|
2482
|
+
const window = options.rescoreWindow ?? k;
|
|
2483
|
+
let depth = window;
|
|
2484
|
+
if (quantizer !== void 0) {
|
|
2485
|
+
depth = options.oversample === void 0 ? defaultRescoreDepth(quantizer.bits, state.dimension, window) : Math.ceil(window * options.oversample);
|
|
2486
|
+
}
|
|
2487
|
+
const ef = Math.max(options.efSearch ?? defaultEfSearch(state.dimension, window), depth);
|
|
2488
|
+
const filter = options.filter;
|
|
2489
|
+
const workspace = state.workspace;
|
|
2490
|
+
const candidates = workspace.traversal;
|
|
2491
|
+
const native = nativeFieldFor();
|
|
2492
|
+
if (native !== null) {
|
|
2493
|
+
candidates.size = 0;
|
|
2494
|
+
if (nativeTraverse(state, native, query, ef, searchMetric, filter, candidates)) {
|
|
2495
|
+
return { candidates, depth: useQuantized ? depth : 0, arenaQuery: null, qMag: 0, native };
|
|
2496
|
+
}
|
|
2497
|
+
}
|
|
2498
|
+
const store = state.store;
|
|
2499
|
+
const arenaQuery = store.prepareQueryArena(query);
|
|
2500
|
+
const qMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
2501
|
+
let distFn;
|
|
2502
|
+
if (quantizer !== void 0) {
|
|
2503
|
+
const prepared = quantizer.prepareQuery(query);
|
|
2504
|
+
if (prepared) distFn = quantizer.preparedDistance(prepared);
|
|
2505
|
+
}
|
|
2506
|
+
if (!distFn && arenaQuery) {
|
|
2507
|
+
distFn = store.queryDistance(arenaQuery, searchMetric);
|
|
2508
|
+
}
|
|
2509
|
+
candidates.size = 0;
|
|
2510
|
+
lockGraphShared(state.locks);
|
|
2511
|
+
try {
|
|
2512
|
+
const entryPoint = entryPointOf(state);
|
|
2513
|
+
if (entryPoint === -1) return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
|
|
2514
|
+
setSingleEntryPoint(workspace, entryPoint);
|
|
2515
|
+
for (let layer = topLayerOf(state); layer >= 1; layer--) {
|
|
2516
|
+
searchLayer(state, query, qMag, 1, layer, searchMetric, false, distFn, candidates);
|
|
2517
|
+
if (candidates.size > 0) {
|
|
2518
|
+
setSingleEntryPoint(workspace, candidates.ords[0]);
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
searchLayer(state, query, qMag, ef, 0, searchMetric, true, distFn, candidates, filter);
|
|
2522
|
+
} finally {
|
|
2523
|
+
unlockGraphShared(state.locks);
|
|
2524
|
+
}
|
|
2525
|
+
return { candidates, depth: useQuantized ? depth : 0, arenaQuery, qMag, native: null };
|
|
2526
|
+
}
|
|
2527
|
+
function bestFirst(a, b) {
|
|
2528
|
+
return b.score - a.score || compareCodePoints(a.docId, b.docId);
|
|
2529
|
+
}
|
|
2530
|
+
function resolveEvery(hits, docIdOf) {
|
|
2531
|
+
const resolved = [];
|
|
2532
|
+
for (const hit of hits) {
|
|
2533
|
+
const docId = docIdOf(hit.ord);
|
|
2534
|
+
if (docId !== void 0) resolved.push({ ord: hit.ord, score: hit.score, docId });
|
|
2535
|
+
}
|
|
2536
|
+
return resolved;
|
|
2537
|
+
}
|
|
2538
|
+
function scoresDescend(hits) {
|
|
2539
|
+
for (let i = 1; i < hits.length; i++) {
|
|
2540
|
+
if (!(hits[i].score <= hits[i - 1].score)) return false;
|
|
2541
|
+
}
|
|
2542
|
+
return hits.length === 0 || !Number.isNaN(hits[0].score);
|
|
2543
|
+
}
|
|
2544
|
+
function bestResolvedHits(hits, k, docIdOf) {
|
|
2545
|
+
if (!Number.isInteger(k) || k <= 0) {
|
|
2546
|
+
return resolveEvery(hits, docIdOf).sort(bestFirst).slice(0, k);
|
|
2547
|
+
}
|
|
2548
|
+
let ordered = hits;
|
|
2549
|
+
if (!scoresDescend(ordered)) {
|
|
2550
|
+
if (hits.some((hit) => Number.isNaN(hit.score))) return resolveEvery(hits, docIdOf).sort(bestFirst).slice(0, k);
|
|
2551
|
+
ordered = hits.slice().sort((a, b) => b.score - a.score);
|
|
2552
|
+
}
|
|
2553
|
+
const taken = [];
|
|
2554
|
+
for (const hit of ordered) {
|
|
2555
|
+
if (taken.length >= k && hit.score < taken[k - 1].score) break;
|
|
2556
|
+
const docId = docIdOf(hit.ord);
|
|
2557
|
+
if (docId !== void 0) taken.push({ ord: hit.ord, score: hit.score, docId });
|
|
2558
|
+
}
|
|
2559
|
+
return taken.sort(bestFirst).slice(0, k);
|
|
2560
|
+
}
|
|
2561
|
+
function collectHits(state, query, k, searchMetric, minSimilarity, options) {
|
|
2562
|
+
if (query.length !== state.dimension) {
|
|
2563
|
+
throw new NarsilError(
|
|
2564
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
2565
|
+
`Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`,
|
|
2566
|
+
{ expected: state.dimension, received: query.length }
|
|
2567
|
+
);
|
|
2568
|
+
}
|
|
2569
|
+
const liveSize2 = nodeCountOf(state) - tombstoneCountOf(state);
|
|
2570
|
+
if (entryPointOf(state) === -1 || liveSize2 === 0) {
|
|
2571
|
+
return [];
|
|
2572
|
+
}
|
|
2573
|
+
const traversal = traverse(state, query, k, searchMetric, options);
|
|
2574
|
+
const { candidates } = traversal;
|
|
2575
|
+
const filter = options.filter;
|
|
2576
|
+
if (traversal.depth === 0) return estimatedHits(candidates, 0, searchMetric, minSimilarity, filter);
|
|
2577
|
+
const rescored = rescoredHits(state, traversal, query, traversal.depth, searchMetric, minSimilarity, filter);
|
|
2578
|
+
if (options.rescoreWindow === void 0 || minSimilarity !== Number.NEGATIVE_INFINITY) return rescored;
|
|
2579
|
+
const rest = endOfRescored(candidates, traversal.depth, filter);
|
|
2580
|
+
return rescored.concat(estimatedHits(candidates, rest, searchMetric, minSimilarity, filter));
|
|
2581
|
+
}
|
|
2582
|
+
function endOfRescored(candidates, depth, filter) {
|
|
2583
|
+
if (filter === void 0) return Math.min(depth, candidates.size);
|
|
2584
|
+
let taken = 0;
|
|
2585
|
+
for (let i = 0; i < candidates.size; i++) {
|
|
2586
|
+
if (!ordinalFilterHas(filter, candidates.ords[i])) continue;
|
|
2587
|
+
taken += 1;
|
|
2588
|
+
if (taken === depth) return i + 1;
|
|
2589
|
+
}
|
|
2590
|
+
return candidates.size;
|
|
2591
|
+
}
|
|
2592
|
+
function estimatedHits(candidates, from, metric, minSimilarity, filter) {
|
|
2593
|
+
const hits = [];
|
|
2594
|
+
for (let i = from; i < candidates.size; i++) {
|
|
2595
|
+
const ord = candidates.ords[i];
|
|
2596
|
+
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
2597
|
+
const score = toScore(candidates.distances[i], metric);
|
|
2598
|
+
if (score < minSimilarity) continue;
|
|
2599
|
+
hits.push({ ord, score });
|
|
2600
|
+
}
|
|
2601
|
+
return hits;
|
|
2602
|
+
}
|
|
2603
|
+
function rescoredHits(state, traversal, query, depth, metric, minSimilarity, filter) {
|
|
2604
|
+
const { candidates, native } = traversal;
|
|
2605
|
+
if (native !== null) {
|
|
2606
|
+
const rescored = nativeRescoredHits(native.store, candidates, query, depth, metric, minSimilarity, filter);
|
|
2607
|
+
if (rescored !== null) return rescored;
|
|
2608
|
+
const arena = state.store.prepareQueryArena(query);
|
|
2609
|
+
const qMag2 = arena ? arena.magnitude : magnitude(query);
|
|
2610
|
+
return rescoreWithFullPrecision(state, candidates, query, qMag2, arena, depth, metric, minSimilarity, filter);
|
|
2611
|
+
}
|
|
2612
|
+
const { qMag, arenaQuery } = traversal;
|
|
2613
|
+
return rescoreWithFullPrecision(state, candidates, query, qMag, arenaQuery, depth, metric, minSimilarity, filter);
|
|
2614
|
+
}
|
|
2615
|
+
function rescoreWithFullPrecision(state, candidates, query, qMag, arenaQuery, depth, metric, minSimilarity, filter) {
|
|
2616
|
+
const rescored = [];
|
|
2617
|
+
let taken = 0;
|
|
2618
|
+
for (let i = 0; i < candidates.size && taken < depth; i++) {
|
|
2619
|
+
const ord = candidates.ords[i];
|
|
2620
|
+
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
2621
|
+
taken += 1;
|
|
2622
|
+
let fullDistance;
|
|
2623
|
+
if (arenaQuery) {
|
|
2624
|
+
fullDistance = state.store.distanceFromArena(arenaQuery, ord, metric);
|
|
2625
|
+
if (fullDistance === Number.POSITIVE_INFINITY) continue;
|
|
2626
|
+
} else {
|
|
2627
|
+
const entry = entryForOrd(state, ord);
|
|
2628
|
+
if (!entry) continue;
|
|
2629
|
+
fullDistance = toDistance(query, entry.vector, qMag, entry.magnitude, metric);
|
|
2630
|
+
}
|
|
2631
|
+
const score = toScore(fullDistance, metric);
|
|
2632
|
+
if (score < minSimilarity) continue;
|
|
2633
|
+
rescored.push({ ord, score });
|
|
2634
|
+
}
|
|
2635
|
+
return rescored;
|
|
2636
|
+
}
|
|
2637
|
+
function search(state, docIdOf, query, k, searchMetric, minSimilarity, options = {}) {
|
|
2638
|
+
const hits = collectHits(state, query, k, searchMetric, minSimilarity, options);
|
|
2639
|
+
return bestResolvedHits(hits, k, docIdOf).map((hit) => ({
|
|
2640
|
+
docId: hit.docId,
|
|
2641
|
+
score: hit.score,
|
|
2642
|
+
termFrequencies: {},
|
|
2643
|
+
fieldLengths: {},
|
|
2644
|
+
idf: {}
|
|
2645
|
+
}));
|
|
2646
|
+
}
|
|
2647
|
+
function searchOrdinals(state, docIdOf, query, k, searchMetric, minSimilarity, options = {}) {
|
|
2648
|
+
const hits = collectHits(state, query, k, searchMetric, minSimilarity, options);
|
|
2649
|
+
return bestResolvedHits(hits, k, docIdOf);
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
// src/vector/hnsw/state.ts
|
|
2653
|
+
function openGraphState(handles, dimension, store, quantizer, threadSlot) {
|
|
2654
|
+
const shape = graphShapeOf(handles.header);
|
|
2655
|
+
const adjacency = openAdjacency(handles, shape.m, shape.mMax0);
|
|
2656
|
+
return {
|
|
2657
|
+
dimension,
|
|
2658
|
+
store,
|
|
2659
|
+
quantizer,
|
|
2660
|
+
adjacency,
|
|
2661
|
+
locks: openGraphLocks(handles, threadSlot),
|
|
2662
|
+
header: handles.header,
|
|
2663
|
+
tombstones: fixedView(handles.tombstones, Uint8Array),
|
|
2664
|
+
visited: new Uint32Array(0),
|
|
2665
|
+
visitStamp: 0,
|
|
2666
|
+
workspace: createHNSWWorkspace(),
|
|
2667
|
+
neighborScratch: new Int32Array(adjacency.level0Stride),
|
|
2668
|
+
M: shape.m,
|
|
2669
|
+
Mmax0: shape.mMax0,
|
|
2670
|
+
efCons: shape.efConstruction,
|
|
2671
|
+
buildMetric: shape.metric,
|
|
2672
|
+
mL: 1 / Math.log(shape.m)
|
|
2673
|
+
};
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
// src/vector/osq/quantize.ts
|
|
2677
|
+
function osqBitsOf(mode) {
|
|
2678
|
+
switch (mode) {
|
|
2679
|
+
case "osq8":
|
|
2680
|
+
return 8;
|
|
2681
|
+
case "osq4":
|
|
2682
|
+
return 4;
|
|
2683
|
+
case "osq2":
|
|
2684
|
+
return 2;
|
|
2685
|
+
case "osq1":
|
|
2686
|
+
return 1;
|
|
2687
|
+
default:
|
|
2688
|
+
return null;
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
function clamp(value, low, high) {
|
|
2692
|
+
return Math.min(Math.max(value, low), high);
|
|
2693
|
+
}
|
|
2694
|
+
function unitNormaliseInto(target, vector) {
|
|
2695
|
+
let squares = 0;
|
|
2696
|
+
for (let i = 0; i < vector.length; i++) squares += vector[i] * vector[i];
|
|
2697
|
+
const length = Math.sqrt(squares);
|
|
2698
|
+
if (length === 0) {
|
|
2699
|
+
for (let i = 0; i < vector.length; i++) target[i] = vector[i];
|
|
2700
|
+
return;
|
|
2701
|
+
}
|
|
2702
|
+
for (let i = 0; i < vector.length; i++) target[i] = vector[i] / length;
|
|
2703
|
+
}
|
|
2704
|
+
var CENTROID_TOTALS_PAST_DIMENSION = 2;
|
|
2705
|
+
function createCentroidSums(dimension, metric) {
|
|
2706
|
+
return {
|
|
2707
|
+
metric,
|
|
2708
|
+
dimension,
|
|
2709
|
+
totals: new Float64Array(dimension + CENTROID_TOTALS_PAST_DIMENSION),
|
|
2710
|
+
scratch: new Float64Array(dimension)
|
|
2711
|
+
};
|
|
2712
|
+
}
|
|
2713
|
+
function countOfCentroidSums(target) {
|
|
2714
|
+
return target.totals[target.dimension + 1];
|
|
2715
|
+
}
|
|
2716
|
+
function squaredLengthsOfCentroidSums(target) {
|
|
2717
|
+
return target.totals[target.dimension];
|
|
2718
|
+
}
|
|
2719
|
+
function addToCentroidSums(target, vector) {
|
|
2720
|
+
const { totals, scratch, dimension } = target;
|
|
2721
|
+
let squares = totals[dimension];
|
|
2722
|
+
if (target.metric === "cosine") {
|
|
2723
|
+
unitNormaliseInto(scratch, vector);
|
|
2724
|
+
for (let i = 0; i < dimension; i++) {
|
|
2725
|
+
totals[i] += scratch[i];
|
|
2726
|
+
squares += scratch[i] * scratch[i];
|
|
2727
|
+
}
|
|
2728
|
+
} else {
|
|
2729
|
+
for (let i = 0; i < dimension; i++) {
|
|
2730
|
+
totals[i] += vector[i];
|
|
2731
|
+
squares += vector[i] * vector[i];
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
totals[dimension] = squares;
|
|
2735
|
+
totals[dimension + 1] += 1;
|
|
2736
|
+
}
|
|
2737
|
+
function meanOfCentroidSums(target) {
|
|
2738
|
+
const count = countOfCentroidSums(target);
|
|
2739
|
+
const mean = new Float64Array(target.dimension);
|
|
2740
|
+
for (let i = 0; i < mean.length; i++) mean[i] = target.totals[i] / count;
|
|
2741
|
+
return mean;
|
|
2742
|
+
}
|
|
2743
|
+
function centroidOfCentroidSums(target) {
|
|
2744
|
+
if (countOfCentroidSums(target) === 0) return null;
|
|
2745
|
+
const dimension = target.dimension;
|
|
2746
|
+
const mean = meanOfCentroidSums(target);
|
|
2747
|
+
const centroid = new Float32Array(dimension);
|
|
2748
|
+
if (target.metric === "cosine") {
|
|
2749
|
+
let squares = 0;
|
|
2750
|
+
for (let i = 0; i < dimension; i++) squares += mean[i] * mean[i];
|
|
2751
|
+
const length = Math.sqrt(squares);
|
|
2752
|
+
for (let i = 0; i < dimension; i++) centroid[i] = length === 0 ? mean[i] : mean[i] / length;
|
|
2753
|
+
return centroid;
|
|
2754
|
+
}
|
|
2755
|
+
for (let i = 0; i < dimension; i++) centroid[i] = mean[i];
|
|
2756
|
+
return centroid;
|
|
2757
|
+
}
|
|
2758
|
+
function level(value, lower, upper, steps) {
|
|
2759
|
+
if (upper === lower) return 0;
|
|
2760
|
+
return Math.floor((clamp(value, lower, upper) - lower) * steps / (upper - lower) + 0.5);
|
|
2761
|
+
}
|
|
2762
|
+
function loss(x, lower, upper, steps, norm) {
|
|
2763
|
+
let xe = 0;
|
|
2764
|
+
let e = 0;
|
|
2765
|
+
const width = (upper - lower) / steps;
|
|
2766
|
+
for (let i = 0; i < x.length; i++) {
|
|
2767
|
+
const q = lower + width * level(x[i], lower, upper, steps);
|
|
2768
|
+
xe += x[i] * (x[i] - q);
|
|
2769
|
+
e += (x[i] - q) * (x[i] - q);
|
|
2770
|
+
}
|
|
2771
|
+
return (1 - OSQ_LAMBDA) * xe * xe / norm + OSQ_LAMBDA * e;
|
|
2772
|
+
}
|
|
2773
|
+
function refine(x, lowerStart, upperStart, steps) {
|
|
2774
|
+
let lower = lowerStart;
|
|
2775
|
+
let upper = upperStart;
|
|
2776
|
+
let norm = 0;
|
|
2777
|
+
for (let i = 0; i < x.length; i++) norm += x[i] * x[i];
|
|
2778
|
+
if (norm === 0 || upper === lower) return [lower, upper];
|
|
2779
|
+
let best = loss(x, lower, upper, steps, norm);
|
|
2780
|
+
for (let round = 0; round < OSQ_REFINE_ROUNDS; round++) {
|
|
2781
|
+
let daa = 0;
|
|
2782
|
+
let dab = 0;
|
|
2783
|
+
let dbb = 0;
|
|
2784
|
+
let dax = 0;
|
|
2785
|
+
let dbx = 0;
|
|
2786
|
+
for (let i = 0; i < x.length; i++) {
|
|
2787
|
+
const s = level(x[i], lower, upper, steps) / steps;
|
|
2788
|
+
daa += (1 - s) * (1 - s);
|
|
2789
|
+
dab += (1 - s) * s;
|
|
2790
|
+
dbb += s * s;
|
|
2791
|
+
dax += x[i] * (1 - s);
|
|
2792
|
+
dbx += x[i] * s;
|
|
2793
|
+
}
|
|
2794
|
+
const m0 = (1 - OSQ_LAMBDA) * dax * dax / norm + OSQ_LAMBDA * daa;
|
|
2795
|
+
const m1 = (1 - OSQ_LAMBDA) * dax * dbx / norm + OSQ_LAMBDA * dab;
|
|
2796
|
+
const m2 = (1 - OSQ_LAMBDA) * dbx * dbx / norm + OSQ_LAMBDA * dbb;
|
|
2797
|
+
const det = m0 * m2 - m1 * m1;
|
|
2798
|
+
if (det === 0) return [lower, upper];
|
|
2799
|
+
const a = (m2 * dax - m1 * dbx) / det;
|
|
2800
|
+
const b = (m0 * dbx - m1 * dax) / det;
|
|
2801
|
+
if (Math.abs(a - lower) < OSQ_REFINE_TOLERANCE && Math.abs(b - upper) < OSQ_REFINE_TOLERANCE) return [lower, upper];
|
|
2802
|
+
const candidate = loss(x, a, b, steps, norm);
|
|
2803
|
+
if (candidate > best) return [lower, upper];
|
|
2804
|
+
lower = a;
|
|
2805
|
+
upper = b;
|
|
2806
|
+
best = candidate;
|
|
2807
|
+
}
|
|
2808
|
+
return [lower, upper];
|
|
2809
|
+
}
|
|
2810
|
+
function createOsqScratch(dimension) {
|
|
2811
|
+
return { normalised: new Float64Array(dimension), centred: new Float64Array(dimension) };
|
|
2812
|
+
}
|
|
2813
|
+
function osqQuantize(vector, centroid, bits, metric, scratch) {
|
|
2814
|
+
const dimension = vector.length;
|
|
2815
|
+
const v = scratch.normalised;
|
|
2816
|
+
if (metric === "cosine") unitNormaliseInto(v, vector);
|
|
2817
|
+
else for (let i = 0; i < dimension; i++) v[i] = vector[i];
|
|
2818
|
+
const x = scratch.centred;
|
|
2819
|
+
let correction = 0;
|
|
2820
|
+
let total = 0;
|
|
2821
|
+
let minimum = Number.POSITIVE_INFINITY;
|
|
2822
|
+
let maximum = Number.NEGATIVE_INFINITY;
|
|
2823
|
+
for (let i = 0; i < dimension; i++) {
|
|
2824
|
+
x[i] = v[i] - centroid[i];
|
|
2825
|
+
correction += metric === "euclidean" ? x[i] * x[i] : v[i] * centroid[i];
|
|
2826
|
+
total += x[i];
|
|
2827
|
+
if (x[i] < minimum) minimum = x[i];
|
|
2828
|
+
if (x[i] > maximum) maximum = x[i];
|
|
2829
|
+
}
|
|
2830
|
+
const mean = total / dimension;
|
|
2831
|
+
let spread = 0;
|
|
2832
|
+
for (let i = 0; i < dimension; i++) spread += (x[i] - mean) * (x[i] - mean);
|
|
2833
|
+
const std = Math.sqrt(spread / dimension);
|
|
2834
|
+
const steps = OSQ_STEPS[bits];
|
|
2835
|
+
const [lower, upper] = refine(
|
|
2836
|
+
x,
|
|
2837
|
+
clamp(mean - OSQ_GRID[bits] * std, minimum, maximum),
|
|
2838
|
+
clamp(mean + OSQ_GRID[bits] * std, minimum, maximum),
|
|
2839
|
+
steps
|
|
2840
|
+
);
|
|
2841
|
+
const levels = new Uint8Array(dimension);
|
|
2842
|
+
let sum = 0;
|
|
2843
|
+
for (let i = 0; i < dimension; i++) {
|
|
2844
|
+
levels[i] = level(x[i], lower, upper, steps);
|
|
2845
|
+
sum += levels[i];
|
|
2846
|
+
}
|
|
2847
|
+
return { levels, lower, upper, correction, sum };
|
|
2848
|
+
}
|
|
2849
|
+
|
|
2850
|
+
// src/vector/osq/products.ts
|
|
2851
|
+
function simdQueryProducts(simd, layout, documentOffset, queryOffset) {
|
|
2852
|
+
const { bits, dimension, codeBytes } = layout;
|
|
2853
|
+
if (bits === 8) return simd.dot_u8(documentOffset, queryOffset, dimension);
|
|
2854
|
+
if (bits === 4) return simd.osq_dot_nibbles_4x4(documentOffset, queryOffset, codeBytes);
|
|
2855
|
+
return simd.osq_dot_bits(documentOffset, queryOffset, codeBytes, bits);
|
|
2856
|
+
}
|
|
2857
|
+
function scalarQueryProducts(layout, document2, documentOffset, packed) {
|
|
2858
|
+
const { bits, dimension, codeBytes } = layout;
|
|
2859
|
+
if (bits === 8) return osqByteLevelProducts(document2, documentOffset, packed, 0, dimension);
|
|
2860
|
+
if (bits === 4) return osqNibbleProducts(document2, documentOffset, packed, 0, codeBytes);
|
|
2861
|
+
return osqStagedQueryProducts(document2, documentOffset, bits, packed, 0, codeBytes);
|
|
2862
|
+
}
|
|
2863
|
+
function simdPairProducts(simd, layout, offsetA, offsetB) {
|
|
2864
|
+
const { bits, dimension, codeBytes } = layout;
|
|
2865
|
+
if (bits === 8) return simd.dot_u8(offsetA, offsetB, dimension);
|
|
2866
|
+
if (bits === 4) return simd.osq_dot_nibbles_4x4(offsetA, offsetB, codeBytes);
|
|
2867
|
+
return simd.osq_dot_bits_pair(offsetA, offsetB, codeBytes, bits);
|
|
2868
|
+
}
|
|
2869
|
+
function scalarPairProducts(layout, bytesA, offsetA, bytesB, offsetB) {
|
|
2870
|
+
const { bits, dimension, codeBytes } = layout;
|
|
2871
|
+
if (bits === 8) return osqByteLevelProducts(bytesA, offsetA, bytesB, offsetB, dimension);
|
|
2872
|
+
if (bits === 4) return osqNibbleProducts(bytesA, offsetA, bytesB, offsetB, codeBytes);
|
|
2873
|
+
return osqNarrowPairProducts(bytesA, offsetA, bytesB, offsetB, bits, codeBytes);
|
|
2874
|
+
}
|
|
2875
|
+
|
|
2876
|
+
// src/vector/osq/recoding.ts
|
|
2877
|
+
function createCodeRecoding(source, centroid) {
|
|
2878
|
+
const { store, layout, metric } = source;
|
|
2879
|
+
const { capacity, slotStride, bits } = layout;
|
|
2880
|
+
const scratch = createOsqScratch(layout.dimension);
|
|
2881
|
+
const blocks = [];
|
|
2882
|
+
let recoded = new Uint8Array(0);
|
|
2883
|
+
function blockReaching(blockIndex, end) {
|
|
2884
|
+
const held = blocks[blockIndex];
|
|
2885
|
+
if (held !== void 0 && held.bytes.length >= end) return held;
|
|
2886
|
+
const length = Math.min(capacity * slotStride, Math.max(end, (held?.bytes.length ?? 0) * 2));
|
|
2887
|
+
const bytes = new Uint8Array(length);
|
|
2888
|
+
if (held !== void 0) bytes.set(held.bytes);
|
|
2889
|
+
const block = { bytes, data: new DataView(bytes.buffer), usedBytes: held?.usedBytes ?? 0 };
|
|
2890
|
+
blocks[blockIndex] = block;
|
|
2891
|
+
return block;
|
|
2892
|
+
}
|
|
2893
|
+
function markRecoded(ordinal) {
|
|
2894
|
+
if (ordinal >= recoded.length) {
|
|
2895
|
+
const grown = new Uint8Array(Math.max(ordinal + 1, recoded.length * 2));
|
|
2896
|
+
grown.set(recoded);
|
|
2897
|
+
recoded = grown;
|
|
2898
|
+
}
|
|
2899
|
+
recoded[ordinal] = 1;
|
|
2900
|
+
}
|
|
2901
|
+
function recode(ordinal) {
|
|
2902
|
+
if (!source.holds(ordinal) || !store.holdsOrdinal(ordinal)) return;
|
|
2903
|
+
const offset = ordinal % capacity * slotStride;
|
|
2904
|
+
const block = blockReaching(Math.floor(ordinal / capacity), offset + slotStride);
|
|
2905
|
+
const code = osqQuantize(store.vectorAt(ordinal), centroid, bits, metric, scratch);
|
|
2906
|
+
writeRecord(block.bytes, offset, code, bits, block.data);
|
|
2907
|
+
block.usedBytes = Math.max(block.usedBytes, offset + slotStride);
|
|
2908
|
+
markRecoded(ordinal);
|
|
2909
|
+
}
|
|
2910
|
+
function copyIntoStore(slots) {
|
|
2911
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
2912
|
+
if (!source.holds(ordinal) || ordinal < recoded.length && recoded[ordinal] === 1) continue;
|
|
2913
|
+
recode(ordinal);
|
|
2914
|
+
if (ordinal >= recoded.length || recoded[ordinal] !== 1) source.clearCodes(ordinal);
|
|
2915
|
+
}
|
|
2916
|
+
const start = slotByteOffset(layout, 0);
|
|
2917
|
+
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
|
|
2918
|
+
const block = blocks[blockIndex];
|
|
2919
|
+
if (block === void 0 || block.usedBytes === 0) continue;
|
|
2920
|
+
const target = store.codeBlockOf(blockIndex * capacity).bytes(start + block.usedBytes);
|
|
2921
|
+
target.set(block.bytes.subarray(0, block.usedBytes), start);
|
|
2922
|
+
}
|
|
2923
|
+
}
|
|
2924
|
+
return { centroid, recode, copyIntoStore };
|
|
2925
|
+
}
|
|
2926
|
+
|
|
2927
|
+
// src/vector/osq/view.ts
|
|
2928
|
+
function openSharedQuantizer(store, layout, metric) {
|
|
2929
|
+
const { dimension, bits, queryBits, codeBytes, capacity } = layout;
|
|
2930
|
+
const scratch = createOsqScratch(dimension);
|
|
2931
|
+
const placementQueryBits = osqPlacementQueryBits(bits);
|
|
2932
|
+
const documentTrailer = { lower: 0, upper: 0, correction: 0, sum: 0 };
|
|
2933
|
+
const otherTrailer = { lower: 0, upper: 0, correction: 0, sum: 0 };
|
|
2934
|
+
let codePresent = fixedView(store.handles.codePresent, Uint8Array);
|
|
2935
|
+
let centroidDot = 0;
|
|
2936
|
+
let centroidGeneration = -1;
|
|
2937
|
+
let currentQuery = null;
|
|
2938
|
+
let recoding = null;
|
|
2939
|
+
const singleOrdinal = new Int32Array(1);
|
|
2940
|
+
function reach2(ordinal) {
|
|
2941
|
+
if (ordinal < codePresent.length) return true;
|
|
2942
|
+
if (ordinal >= store.handles.codePresent.byteLength) return false;
|
|
2943
|
+
codePresent = fixedView(store.handles.codePresent, Uint8Array);
|
|
2944
|
+
return ordinal < codePresent.length;
|
|
2945
|
+
}
|
|
2946
|
+
function isCalibrated() {
|
|
2947
|
+
return Atomics.load(store.handles.header, STORE_CALIBRATED) === 1;
|
|
2948
|
+
}
|
|
2949
|
+
function recomputeCentroidDot(generation) {
|
|
2950
|
+
const centroid = store.handles.centroid;
|
|
2951
|
+
let total = 0;
|
|
2952
|
+
for (let i = 0; i < dimension; i++) total += centroid[i] * centroid[i];
|
|
2953
|
+
centroidDot = total;
|
|
2954
|
+
centroidGeneration = generation;
|
|
2955
|
+
}
|
|
2956
|
+
function centroidDotNow() {
|
|
2957
|
+
const generation = Atomics.load(store.handles.header, STORE_CALIBRATION_GENERATION);
|
|
2958
|
+
if (generation !== centroidGeneration) recomputeCentroidDot(generation);
|
|
2959
|
+
return centroidDot;
|
|
2960
|
+
}
|
|
2961
|
+
function holds2(ordinal) {
|
|
2962
|
+
return ordinal >= 0 && reach2(ordinal) && codePresent[ordinal] === 1;
|
|
2963
|
+
}
|
|
2964
|
+
function recordOffset(ordinal) {
|
|
2965
|
+
return slotByteOffset(layout, ordinal % capacity);
|
|
2966
|
+
}
|
|
2967
|
+
function markPresent(ordinal) {
|
|
2968
|
+
if (Atomics.load(codePresent, ordinal) !== 0) return;
|
|
2969
|
+
Atomics.add(store.handles.header, STORE_CODE_COUNT, 1);
|
|
2970
|
+
Atomics.store(codePresent, ordinal, 1);
|
|
2971
|
+
}
|
|
2972
|
+
function writeCentroid(centroid) {
|
|
2973
|
+
store.handles.centroid.set(centroid);
|
|
2974
|
+
Atomics.add(store.handles.header, STORE_CALIBRATION_GENERATION, 1);
|
|
2975
|
+
Atomics.store(store.handles.header, STORE_CALIBRATED, 1);
|
|
2976
|
+
}
|
|
2977
|
+
function measureInto(ordinals, sums) {
|
|
2978
|
+
if (nativeMeasure(store.handles, metric, ordinals, sums.totals)) return;
|
|
2979
|
+
for (const vector of storedVectors(ordinals)) addToCentroidSums(sums, vector);
|
|
2980
|
+
}
|
|
2981
|
+
function* storedVectors(ordinals) {
|
|
2982
|
+
for (const ordinal of ordinals) {
|
|
2983
|
+
if (store.holdsOrdinal(ordinal)) yield store.vectorAt(ordinal);
|
|
2984
|
+
}
|
|
2985
|
+
}
|
|
2986
|
+
function writeRecordOf(ordinal) {
|
|
2987
|
+
if (!reach2(ordinal) || !store.holdsOrdinal(ordinal)) return;
|
|
2988
|
+
const code = osqQuantize(store.vectorAt(ordinal), store.handles.centroid, bits, metric, scratch);
|
|
2989
|
+
const block = store.codeBlockOf(ordinal);
|
|
2990
|
+
const offset = recordOffset(ordinal);
|
|
2991
|
+
writeRecord(block.bytes(offset + layout.slotStride), offset, code, bits, block.data(offset + layout.slotStride));
|
|
2992
|
+
markPresent(ordinal);
|
|
2993
|
+
recoding?.recode(ordinal);
|
|
2994
|
+
}
|
|
2995
|
+
function writeCodesOf(ordinals) {
|
|
2996
|
+
if (!isCalibrated() || ordinals.length === 0) return;
|
|
2997
|
+
if (nativeQuantise(store.handles, metric, ordinals)) {
|
|
2998
|
+
if (recoding !== null) for (const ordinal of ordinals) recoding.recode(ordinal);
|
|
2999
|
+
return;
|
|
3000
|
+
}
|
|
3001
|
+
for (const ordinal of ordinals) writeRecordOf(ordinal);
|
|
3002
|
+
}
|
|
3003
|
+
function clearCodes(ordinal) {
|
|
3004
|
+
if (ordinal < 0 || !reach2(ordinal)) return;
|
|
3005
|
+
if (Atomics.exchange(codePresent, ordinal, 0) === 1) Atomics.sub(store.handles.header, STORE_CODE_COUNT, 1);
|
|
3006
|
+
}
|
|
3007
|
+
function stageQuery(block, query) {
|
|
3008
|
+
if (block.simd === null || !block.hasScratch) return false;
|
|
3009
|
+
if (block.stagedOrdinal !== QUERY_STAGED) {
|
|
3010
|
+
block.bytes(block.scratchByteOffset + query.packed.length).set(query.packed, block.scratchByteOffset);
|
|
3011
|
+
block.stagedOrdinal = QUERY_STAGED;
|
|
3012
|
+
}
|
|
3013
|
+
return true;
|
|
3014
|
+
}
|
|
3015
|
+
function productsAgainstQuery(block, documentOffset, query) {
|
|
3016
|
+
if (block.simd !== null && stageQuery(block, query)) {
|
|
3017
|
+
return simdQueryProducts(block.simd, layout, documentOffset, block.scratchByteOffset);
|
|
3018
|
+
}
|
|
3019
|
+
return scalarQueryProducts(layout, block.bytes(documentOffset + codeBytes), documentOffset, query.packed);
|
|
3020
|
+
}
|
|
3021
|
+
function distanceFrom(products, query, levelBits) {
|
|
3022
|
+
const estimate = osqEstimate(products, documentTrailer, bits, query, levelBits, dimension, centroidDotNow(), metric);
|
|
3023
|
+
return osqDistance(estimate, metric);
|
|
3024
|
+
}
|
|
3025
|
+
function estimateDistance(ordinal, query) {
|
|
3026
|
+
const block = store.codeBlockOf(ordinal);
|
|
3027
|
+
const offset = recordOffset(ordinal);
|
|
3028
|
+
const products = productsAgainstQuery(block, offset, query);
|
|
3029
|
+
readTrailer(block.data(offset + layout.slotStride), offset, codeBytes, documentTrailer);
|
|
3030
|
+
return distanceFrom(products, query, queryBits);
|
|
3031
|
+
}
|
|
3032
|
+
function prepareQuery(query) {
|
|
3033
|
+
if (!isCalibrated() || query.length !== dimension) return null;
|
|
3034
|
+
const code = osqQuantize(query, store.handles.centroid, queryBits, metric, scratch);
|
|
3035
|
+
const packed = new Uint8Array(layout.queryBytes);
|
|
3036
|
+
stageQueryLevels(code.levels, bits, packed);
|
|
3037
|
+
for (const block of store.openCodeBlocks()) block.stagedOrdinal = NOTHING_STAGED;
|
|
3038
|
+
currentQuery = {
|
|
3039
|
+
packed,
|
|
3040
|
+
lower: code.lower,
|
|
3041
|
+
upper: code.upper,
|
|
3042
|
+
correction: code.correction,
|
|
3043
|
+
sum: code.sum
|
|
3044
|
+
};
|
|
3045
|
+
return currentQuery;
|
|
3046
|
+
}
|
|
3047
|
+
function preparedDistance(prepared) {
|
|
3048
|
+
const generic = (ordinal) => holds2(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
|
|
3049
|
+
if (store.handles.codeBlocks.length === 0) return generic;
|
|
3050
|
+
const block = store.codeBlockOf(0);
|
|
3051
|
+
const simd = block.simd;
|
|
3052
|
+
if (simd === null || !block.hasScratch) return generic;
|
|
3053
|
+
const presentView = codePresent;
|
|
3054
|
+
const limit = Math.min(presentView.length, capacity);
|
|
3055
|
+
const { slotsOffset, slotStride } = layout;
|
|
3056
|
+
const data = block.data(slotsOffset + slotStride);
|
|
3057
|
+
const dataLength = data.byteLength;
|
|
3058
|
+
const queryOffset = block.scratchByteOffset;
|
|
3059
|
+
return (ordinal) => {
|
|
3060
|
+
if (ordinal < 0 || ordinal >= limit || block.stagedOrdinal !== QUERY_STAGED) return generic(ordinal);
|
|
3061
|
+
if (presentView[ordinal] !== 1) return Number.POSITIVE_INFINITY;
|
|
3062
|
+
const offset = slotsOffset + ordinal * slotStride;
|
|
3063
|
+
if (offset + slotStride > dataLength) return generic(ordinal);
|
|
3064
|
+
const products = simdQueryProducts(simd, layout, offset, queryOffset);
|
|
3065
|
+
readTrailer(data, offset, codeBytes, documentTrailer);
|
|
3066
|
+
return distanceFrom(products, prepared, queryBits);
|
|
3067
|
+
};
|
|
3068
|
+
}
|
|
3069
|
+
function placementDistance2(vector) {
|
|
3070
|
+
if (placementQueryBits === queryBits) {
|
|
3071
|
+
const prepared = prepareQuery(vector);
|
|
3072
|
+
return prepared === null ? null : preparedDistance(prepared);
|
|
3073
|
+
}
|
|
3074
|
+
if (!isCalibrated() || vector.length !== dimension) return null;
|
|
3075
|
+
const code = osqQuantize(vector, store.handles.centroid, placementQueryBits, metric, scratch);
|
|
3076
|
+
const levels = new Uint8Array(2 * codeBytes);
|
|
3077
|
+
levels.set(code.levels);
|
|
3078
|
+
return (ordinal) => {
|
|
3079
|
+
if (!holds2(ordinal)) return Number.POSITIVE_INFINITY;
|
|
3080
|
+
const block = store.codeBlockOf(ordinal);
|
|
3081
|
+
const offset = recordOffset(ordinal);
|
|
3082
|
+
const products = osqNibblesByByteLevels(block.bytes(offset + codeBytes), offset, levels, codeBytes);
|
|
3083
|
+
readTrailer(block.data(offset + layout.slotStride), offset, codeBytes, documentTrailer);
|
|
3084
|
+
return distanceFrom(products, code, placementQueryBits);
|
|
3085
|
+
};
|
|
3086
|
+
}
|
|
3087
|
+
function distanceBetween(ordA, ordB) {
|
|
3088
|
+
if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
|
|
3089
|
+
const blockA = store.codeBlockOf(ordA);
|
|
3090
|
+
const blockB = store.codeBlockOf(ordB);
|
|
3091
|
+
const offsetA = recordOffset(ordA);
|
|
3092
|
+
const offsetB = recordOffset(ordB);
|
|
3093
|
+
const products = blockA === blockB && blockA.simd !== null ? simdPairProducts(blockA.simd, layout, offsetA, offsetB) : scalarPairProducts(
|
|
3094
|
+
layout,
|
|
3095
|
+
blockA.bytes(offsetA + codeBytes),
|
|
3096
|
+
offsetA,
|
|
3097
|
+
blockB.bytes(offsetB + codeBytes),
|
|
3098
|
+
offsetB
|
|
3099
|
+
);
|
|
3100
|
+
readTrailer(blockA.data(offsetA + layout.slotStride), offsetA, codeBytes, documentTrailer);
|
|
3101
|
+
readTrailer(blockB.data(offsetB + layout.slotStride), offsetB, codeBytes, otherTrailer);
|
|
3102
|
+
return distanceFrom(products, otherTrailer, bits);
|
|
3103
|
+
}
|
|
3104
|
+
function pairDistance() {
|
|
3105
|
+
if (store.handles.codeBlocks.length === 0) return distanceBetween;
|
|
3106
|
+
const block = store.codeBlockOf(0);
|
|
3107
|
+
const simd = block.simd;
|
|
3108
|
+
if (simd === null) return distanceBetween;
|
|
3109
|
+
const presentView = codePresent;
|
|
3110
|
+
const limit = Math.min(presentView.length, capacity);
|
|
3111
|
+
const { slotsOffset, slotStride } = layout;
|
|
3112
|
+
const data = block.data(slotsOffset + slotStride);
|
|
3113
|
+
const dataLength = data.byteLength;
|
|
3114
|
+
return (ordA, ordB) => {
|
|
3115
|
+
if (ordA < 0 || ordB < 0 || ordA >= limit || ordB >= limit) return distanceBetween(ordA, ordB);
|
|
3116
|
+
if (presentView[ordA] !== 1 || presentView[ordB] !== 1) return Number.POSITIVE_INFINITY;
|
|
3117
|
+
const offsetA = slotsOffset + ordA * slotStride;
|
|
3118
|
+
const offsetB = slotsOffset + ordB * slotStride;
|
|
3119
|
+
if (Math.max(offsetA, offsetB) + slotStride > dataLength) return distanceBetween(ordA, ordB);
|
|
3120
|
+
const products = simdPairProducts(simd, layout, offsetA, offsetB);
|
|
3121
|
+
readTrailer(data, offsetA, codeBytes, documentTrailer);
|
|
3122
|
+
readTrailer(data, offsetB, codeBytes, otherTrailer);
|
|
3123
|
+
return distanceFrom(products, otherTrailer, bits);
|
|
3124
|
+
};
|
|
3125
|
+
}
|
|
3126
|
+
return {
|
|
3127
|
+
bits,
|
|
3128
|
+
metric,
|
|
3129
|
+
get size() {
|
|
3130
|
+
return Atomics.load(store.handles.header, STORE_CODE_COUNT);
|
|
3131
|
+
},
|
|
3132
|
+
get centroid() {
|
|
3133
|
+
return isCalibrated() ? store.handles.centroid : null;
|
|
3134
|
+
},
|
|
3135
|
+
isCalibrated,
|
|
3136
|
+
holdsOrdinal: holds2,
|
|
3137
|
+
writeCodes(ordinal) {
|
|
3138
|
+
singleOrdinal[0] = ordinal;
|
|
3139
|
+
writeCodesOf(singleOrdinal);
|
|
3140
|
+
},
|
|
3141
|
+
writeCodesOf,
|
|
3142
|
+
measureInto,
|
|
3143
|
+
calibrateFrom(ordinals) {
|
|
3144
|
+
const sums = createCentroidSums(dimension, metric);
|
|
3145
|
+
measureInto(ordinals, sums);
|
|
3146
|
+
const centroid = centroidOfCentroidSums(sums);
|
|
3147
|
+
if (centroid === null) return false;
|
|
3148
|
+
writeCentroid(centroid);
|
|
3149
|
+
return true;
|
|
3150
|
+
},
|
|
3151
|
+
restoreRecord(ordinal, record) {
|
|
3152
|
+
if (!reach2(ordinal) || record.length !== layout.slotStride) return;
|
|
3153
|
+
const block = store.codeBlockOf(ordinal);
|
|
3154
|
+
const offset = recordOffset(ordinal);
|
|
3155
|
+
block.bytes(offset + layout.slotStride).set(record, offset);
|
|
3156
|
+
markPresent(ordinal);
|
|
3157
|
+
recoding?.recode(ordinal);
|
|
3158
|
+
},
|
|
3159
|
+
recordAt(ordinal) {
|
|
3160
|
+
const block = store.codeBlockOf(ordinal);
|
|
3161
|
+
const offset = recordOffset(ordinal);
|
|
3162
|
+
return block.bytes(offset + layout.slotStride).subarray(offset, offset + layout.slotStride);
|
|
3163
|
+
},
|
|
3164
|
+
clearCodes,
|
|
3165
|
+
writeCentroid,
|
|
3166
|
+
resetCalibration() {
|
|
3167
|
+
recoding = null;
|
|
3168
|
+
Atomics.store(store.handles.header, STORE_CALIBRATED, 0);
|
|
3169
|
+
store.handles.centroid.fill(0);
|
|
3170
|
+
Atomics.add(store.handles.header, STORE_CALIBRATION_GENERATION, 1);
|
|
3171
|
+
},
|
|
3172
|
+
resetCodes() {
|
|
3173
|
+
recoding = null;
|
|
3174
|
+
codePresent = fixedView(store.handles.codePresent, Uint8Array);
|
|
3175
|
+
codePresent.fill(0);
|
|
3176
|
+
Atomics.store(store.handles.header, STORE_CODE_COUNT, 0);
|
|
3177
|
+
},
|
|
3178
|
+
beginRecoding(centroid) {
|
|
3179
|
+
recoding = createCodeRecoding({ store, layout, metric, holds: holds2, clearCodes }, Float32Array.from(centroid));
|
|
3180
|
+
},
|
|
3181
|
+
recode(ordinal) {
|
|
3182
|
+
recoding?.recode(ordinal);
|
|
3183
|
+
},
|
|
3184
|
+
finishRecoding() {
|
|
3185
|
+
if (recoding === null) return false;
|
|
3186
|
+
const finished = recoding;
|
|
3187
|
+
recoding = null;
|
|
3188
|
+
finished.copyIntoStore(store.slots);
|
|
3189
|
+
writeCentroid(finished.centroid);
|
|
3190
|
+
return true;
|
|
3191
|
+
},
|
|
3192
|
+
abandonRecoding() {
|
|
3193
|
+
recoding = null;
|
|
3194
|
+
},
|
|
3195
|
+
prepareQuery,
|
|
3196
|
+
distanceFromPreparedByOrdinal(prepared, ordinal) {
|
|
3197
|
+
if (!holds2(ordinal)) return Number.POSITIVE_INFINITY;
|
|
3198
|
+
return estimateDistance(ordinal, prepared);
|
|
3199
|
+
},
|
|
3200
|
+
preparedDistance,
|
|
3201
|
+
pairDistance,
|
|
3202
|
+
placementDistance: placementDistance2
|
|
3203
|
+
};
|
|
3204
|
+
}
|
|
3205
|
+
|
|
3206
|
+
// src/vector/osq/index.ts
|
|
3207
|
+
function createOsqQuantizer(dimension, bits, metric, store) {
|
|
3208
|
+
let opened = null;
|
|
3209
|
+
function shared() {
|
|
3210
|
+
if (opened === null && store.dimension === dimension && store.handles.codeLayout !== null) {
|
|
3211
|
+
opened = openSharedQuantizer(store.view, store.handles.codeLayout, metric);
|
|
3212
|
+
}
|
|
3213
|
+
return opened;
|
|
3214
|
+
}
|
|
3215
|
+
function requireShared() {
|
|
3216
|
+
const view = shared();
|
|
3217
|
+
if (view === null) throw new Error("The vector store holds no vector for the quantizer to work on");
|
|
3218
|
+
return view;
|
|
3219
|
+
}
|
|
3220
|
+
function ordinalOf(docId) {
|
|
3221
|
+
const ordinal = store.getOrdinal(docId);
|
|
3222
|
+
if (ordinal === void 0) {
|
|
3223
|
+
throw new Error(`Cannot quantize "${docId}": the vector store holds no vector for it`);
|
|
3224
|
+
}
|
|
3225
|
+
return ordinal;
|
|
3226
|
+
}
|
|
3227
|
+
return {
|
|
3228
|
+
dimension,
|
|
3229
|
+
bits,
|
|
3230
|
+
metric,
|
|
3231
|
+
get size() {
|
|
3232
|
+
return shared()?.size ?? 0;
|
|
3233
|
+
},
|
|
3234
|
+
get centroid() {
|
|
3235
|
+
return shared()?.centroid ?? null;
|
|
3236
|
+
},
|
|
3237
|
+
isCalibrated: () => shared()?.isCalibrated() ?? false,
|
|
3238
|
+
calibrate(ordinals) {
|
|
3239
|
+
if (ordinals.length > 0) requireShared().calibrateFrom(ordinals);
|
|
3240
|
+
},
|
|
3241
|
+
measure(ordinals, sums) {
|
|
3242
|
+
if (ordinals.length > 0) requireShared().measureInto(ordinals, sums);
|
|
3243
|
+
},
|
|
3244
|
+
quantize(docId) {
|
|
3245
|
+
const ordinal = ordinalOf(docId);
|
|
3246
|
+
const view = requireShared();
|
|
3247
|
+
if (!view.isCalibrated()) view.calibrateFrom(Int32Array.of(ordinal));
|
|
3248
|
+
view.writeCodes(ordinal);
|
|
3249
|
+
},
|
|
3250
|
+
writeCodes(ordinal) {
|
|
3251
|
+
shared()?.writeCodes(ordinal);
|
|
3252
|
+
},
|
|
3253
|
+
remove(docId) {
|
|
3254
|
+
const ordinal = store.getOrdinal(docId);
|
|
3255
|
+
if (ordinal !== void 0) shared()?.clearCodes(ordinal);
|
|
3256
|
+
},
|
|
3257
|
+
removeOrdinal(ordinal) {
|
|
3258
|
+
shared()?.clearCodes(ordinal);
|
|
3259
|
+
},
|
|
3260
|
+
hasOrdinal: (ordinal) => shared()?.holdsOrdinal(ordinal) ?? false,
|
|
3261
|
+
getLevels(docId) {
|
|
3262
|
+
const view = shared();
|
|
3263
|
+
const ordinal = store.getOrdinal(docId);
|
|
3264
|
+
if (view === null || ordinal === void 0 || !view.holdsOrdinal(ordinal)) return void 0;
|
|
3265
|
+
return unpackLevels(view.recordAt(ordinal), 0, dimension, bits);
|
|
3266
|
+
},
|
|
3267
|
+
recordOf(docId) {
|
|
3268
|
+
const view = shared();
|
|
3269
|
+
const ordinal = store.getOrdinal(docId);
|
|
3270
|
+
if (view === null || ordinal === void 0 || !view.holdsOrdinal(ordinal)) return void 0;
|
|
3271
|
+
return view.recordAt(ordinal);
|
|
3272
|
+
},
|
|
3273
|
+
recordAtOrdinal(ordinal) {
|
|
3274
|
+
const view = shared();
|
|
3275
|
+
if (view === null || !view.holdsOrdinal(ordinal)) return void 0;
|
|
3276
|
+
return view.recordAt(ordinal);
|
|
3277
|
+
},
|
|
3278
|
+
restoreRecord(docId, record) {
|
|
3279
|
+
requireShared().restoreRecord(ordinalOf(docId), record);
|
|
3280
|
+
},
|
|
3281
|
+
restoreRecordAtOrdinal(ordinal, record) {
|
|
3282
|
+
requireShared().restoreRecord(ordinal, record);
|
|
3283
|
+
},
|
|
3284
|
+
restoreCentroid(centroid) {
|
|
3285
|
+
requireShared().writeCentroid(centroid);
|
|
3286
|
+
},
|
|
3287
|
+
recalibrate(ordinals) {
|
|
3288
|
+
if (ordinals.length === 0) return;
|
|
3289
|
+
const view = requireShared();
|
|
3290
|
+
view.resetCodes();
|
|
3291
|
+
view.calibrateFrom(ordinals);
|
|
3292
|
+
view.writeCodesOf(ordinals);
|
|
3293
|
+
},
|
|
3294
|
+
beginRecoding(centroid) {
|
|
3295
|
+
requireShared().beginRecoding(centroid);
|
|
3296
|
+
},
|
|
3297
|
+
recode(ordinal) {
|
|
3298
|
+
shared()?.recode(ordinal);
|
|
3299
|
+
},
|
|
3300
|
+
finishRecoding: () => shared()?.finishRecoding() ?? false,
|
|
3301
|
+
abandonRecoding() {
|
|
3302
|
+
shared()?.abandonRecoding();
|
|
3303
|
+
},
|
|
3304
|
+
prepareQuery: (query) => shared()?.prepareQuery(query) ?? null,
|
|
3305
|
+
distanceFromPreparedByOrdinal: (prepared, ordinal) => shared()?.distanceFromPreparedByOrdinal(prepared, ordinal) ?? Number.POSITIVE_INFINITY,
|
|
3306
|
+
preparedDistance(prepared) {
|
|
3307
|
+
const view = shared();
|
|
3308
|
+
if (view === null) return () => Number.POSITIVE_INFINITY;
|
|
3309
|
+
return view.preparedDistance(prepared);
|
|
3310
|
+
},
|
|
3311
|
+
pairDistance() {
|
|
3312
|
+
const view = shared();
|
|
3313
|
+
if (view === null) return () => Number.POSITIVE_INFINITY;
|
|
3314
|
+
return view.pairDistance();
|
|
3315
|
+
},
|
|
3316
|
+
placementDistance: (vector) => shared()?.placementDistance(vector) ?? null,
|
|
3317
|
+
clear() {
|
|
3318
|
+
const view = shared();
|
|
3319
|
+
if (view === null) return;
|
|
3320
|
+
view.resetCodes();
|
|
3321
|
+
view.resetCalibration();
|
|
3322
|
+
}
|
|
3323
|
+
};
|
|
3324
|
+
}
|
|
3325
|
+
|
|
3326
|
+
// src/vector/vector-index/constants.ts
|
|
3327
|
+
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
3328
|
+
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
3329
|
+
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
3330
|
+
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
3331
|
+
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
3332
|
+
var BUILD_CHUNK_SIZE = 100;
|
|
3333
|
+
var BUILD_TURN_HANDOVER_AFTER_MS = 50;
|
|
3334
|
+
var BUILD_RETRY_FIRST_DELAY_MS = 250;
|
|
3335
|
+
var BUILD_RETRY_MAX_DELAY_MS = 3e4;
|
|
3336
|
+
var REBUILD_REMOVED_RATIO = 0.2;
|
|
3337
|
+
var DRIFT_CHECK_GROWTH_FACTOR = 2;
|
|
3338
|
+
var RECALIBRATION_DRIFT_RATIO = 2.5;
|
|
3339
|
+
var RECALIBRATION_CHUNK_VECTORS = 512;
|
|
3340
|
+
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
3341
|
+
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
3342
|
+
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
3343
|
+
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
3344
|
+
var OSQ4_MIN_DIMENSION = 384;
|
|
3345
|
+
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
3346
|
+
|
|
3347
|
+
// src/vector/vector-index/shared.ts
|
|
3348
|
+
var VECTOR_WORKER_COPIES_ALLOWED = { enabled: true };
|
|
3349
|
+
function liveSize(state) {
|
|
3350
|
+
return state.store.size - state.tombstones.size;
|
|
3351
|
+
}
|
|
3352
|
+
function threadsHoldCurrentLayout(state) {
|
|
3353
|
+
return state.sharedLayoutRevision === state.store.handles.layoutRevision;
|
|
3354
|
+
}
|
|
3355
|
+
function assignStorePartitions(state, resolve) {
|
|
3356
|
+
for (let ordinal = 0; ordinal < state.store.slots; ordinal += 1) {
|
|
3357
|
+
if (state.store.partitionOfOrdinal(ordinal) !== void 0) continue;
|
|
3358
|
+
const docId = state.store.docIdForOrdinal(ordinal);
|
|
3359
|
+
if (docId === void 0) continue;
|
|
3360
|
+
const partitionId = resolve(docId);
|
|
3361
|
+
if (partitionId === void 0) {
|
|
3362
|
+
state.store.forgetPartition(docId);
|
|
3363
|
+
continue;
|
|
3364
|
+
}
|
|
3365
|
+
state.store.setPartition(docId, partitionId);
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
function emptyFieldBeforeRestore(state) {
|
|
3369
|
+
state.store.clear();
|
|
3370
|
+
state.tombstones.clear();
|
|
3371
|
+
state.buffer.clear();
|
|
3372
|
+
state.pendingLocations.clear();
|
|
3373
|
+
adoptGraph(state, null);
|
|
3374
|
+
state.osq?.clear();
|
|
3375
|
+
forgetCalibration(state);
|
|
3376
|
+
state.savedFiles = [];
|
|
3377
|
+
state.savedSignature = null;
|
|
3378
|
+
}
|
|
3379
|
+
function releaseReplacedVectors(state) {
|
|
3380
|
+
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
3381
|
+
}
|
|
3382
|
+
function adoptGraph(state, graph) {
|
|
3383
|
+
if (state.hnsw !== null && state.hnsw !== graph) state.hnsw.retire();
|
|
3384
|
+
if (graph === null && state.hnsw !== null) {
|
|
3385
|
+
state.hnsw.clear();
|
|
3386
|
+
}
|
|
3387
|
+
state.hnsw = graph;
|
|
3388
|
+
state.shuffledBuildCount = null;
|
|
3389
|
+
state.compactedNodeCount = 0;
|
|
3390
|
+
if (graph === null) return;
|
|
3391
|
+
for (const docId of state.tombstones) {
|
|
3392
|
+
if (graph.has(docId)) {
|
|
3393
|
+
graph.markTombstone(docId);
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
function graphNeedsRebuild(state) {
|
|
3398
|
+
const graph = state.hnsw;
|
|
3399
|
+
if (graph === null) return false;
|
|
3400
|
+
const removed = graph.tombstoneCount + state.compactedNodeCount;
|
|
3401
|
+
const held = graph.size + removed;
|
|
3402
|
+
if (held === 0) return false;
|
|
3403
|
+
return removed / held > REBUILD_REMOVED_RATIO;
|
|
3404
|
+
}
|
|
3405
|
+
function yieldBuildTurn(state) {
|
|
3406
|
+
return new Promise((resolve) => {
|
|
3407
|
+
const takeTurn = () => {
|
|
3408
|
+
clearTimeout(timer);
|
|
3409
|
+
state.buildTurnWaiters.delete(takeTurn);
|
|
3410
|
+
resolve();
|
|
3411
|
+
};
|
|
3412
|
+
const timer = setTimeout(takeTurn, 0);
|
|
3413
|
+
state.buildTurnWaiters.set(takeTurn, performance.now());
|
|
3414
|
+
});
|
|
3415
|
+
}
|
|
3416
|
+
function noBuildHistory() {
|
|
3417
|
+
return {
|
|
3418
|
+
shuffledBuildCount: null,
|
|
3419
|
+
calibratedCount: null,
|
|
3420
|
+
calibratedMeanLength: null,
|
|
3421
|
+
nextDriftCheckAt: Number.POSITIVE_INFINITY,
|
|
3422
|
+
shuffledRebuild: null
|
|
3423
|
+
};
|
|
3424
|
+
}
|
|
3425
|
+
function handOverBuildTurns(state) {
|
|
3426
|
+
if (state.buildTurnWaiters.size === 0) return;
|
|
3427
|
+
const [longestWaitingSince] = state.buildTurnWaiters.values();
|
|
3428
|
+
if (longestWaitingSince === void 0 || performance.now() - longestWaitingSince < BUILD_TURN_HANDOVER_AFTER_MS)
|
|
3429
|
+
return;
|
|
3430
|
+
for (const takeTurn of [...state.buildTurnWaiters.keys()]) takeTurn();
|
|
3431
|
+
}
|
|
3432
|
+
function* allLiveDocIds(state) {
|
|
3433
|
+
for (const docId of state.store.docIds()) {
|
|
3434
|
+
if (state.tombstones.has(docId)) continue;
|
|
3435
|
+
yield docId;
|
|
3436
|
+
}
|
|
3437
|
+
}
|
|
3438
|
+
function ordinalFilterForPartitions(state, partitionIds) {
|
|
3439
|
+
const filter = state.store.partitionFilter(partitionIds);
|
|
3440
|
+
for (const docId of state.tombstones) {
|
|
3441
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
3442
|
+
if (ordinal === void 0) continue;
|
|
3443
|
+
removeFromOrdinalFilter(filter, ordinal);
|
|
3444
|
+
}
|
|
3445
|
+
return filter;
|
|
3446
|
+
}
|
|
3447
|
+
function ordinalSpaceOf(state) {
|
|
3448
|
+
const store = state.store;
|
|
3449
|
+
const handles = store.handles;
|
|
3450
|
+
const removed = new Uint8Array(handles.removed);
|
|
3451
|
+
return {
|
|
3452
|
+
fieldName: state.fieldName,
|
|
3453
|
+
slots: store.slots,
|
|
3454
|
+
header: handles.header,
|
|
3455
|
+
layoutGeneration: Atomics.load(handles.header, STORE_ORDINAL_LAYOUT_GENERATION),
|
|
3456
|
+
ordinalOf: (docId) => store.getOrdinal(docId),
|
|
3457
|
+
holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal),
|
|
3458
|
+
isRemoved: (ordinal) => ordinal < removed.length && removed[ordinal] === 1
|
|
3459
|
+
};
|
|
3460
|
+
}
|
|
3461
|
+
function filterForOptions(state, options) {
|
|
3462
|
+
if (options.filterSubsets !== void 0) {
|
|
3463
|
+
if (state.store.dimension === 0) return createOrdinalFilter(0);
|
|
3464
|
+
return ordinalFilterForSubsets(options.filterSubsets, ordinalSpaceOf(state));
|
|
3465
|
+
}
|
|
3466
|
+
if (options.filterPartitions !== void 0) {
|
|
3467
|
+
return ordinalFilterForPartitions(state, options.filterPartitions);
|
|
3468
|
+
}
|
|
3469
|
+
return void 0;
|
|
3470
|
+
}
|
|
3471
|
+
function liveOrdinals(state) {
|
|
3472
|
+
const ordinals = [];
|
|
3473
|
+
for (let ordinal = 0; ordinal < state.store.slots; ordinal++) {
|
|
3474
|
+
const docId = state.store.docIdForOrdinal(ordinal);
|
|
3475
|
+
if (docId !== void 0 && !state.tombstones.has(docId)) ordinals.push(ordinal);
|
|
3476
|
+
}
|
|
3477
|
+
return Int32Array.from(ordinals);
|
|
3478
|
+
}
|
|
3479
|
+
function forgetCalibration(state) {
|
|
3480
|
+
state.calibratedCount = null;
|
|
3481
|
+
state.calibratedMeanLength = null;
|
|
3482
|
+
state.nextDriftCheckAt = Number.POSITIVE_INFINITY;
|
|
3483
|
+
}
|
|
3484
|
+
function adoptLoadedCalibration(state, calibratedCount, calibratedMeanLength) {
|
|
3485
|
+
if (state.osq === null || !state.osq.isCalibrated()) return;
|
|
3486
|
+
state.calibratedCount = calibratedCount;
|
|
3487
|
+
state.calibratedMeanLength = calibratedMeanLength;
|
|
3488
|
+
state.nextDriftCheckAt = Math.max(1, calibratedCount) * DRIFT_CHECK_GROWTH_FACTOR;
|
|
3489
|
+
}
|
|
3490
|
+
function recalibrateFromStore(state) {
|
|
3491
|
+
if (state.osq === null) return;
|
|
3492
|
+
const ordinals = liveOrdinals(state);
|
|
3493
|
+
state.osq.recalibrate(ordinals);
|
|
3494
|
+
state.calibratedCount = ordinals.length;
|
|
3495
|
+
state.calibratedMeanLength = null;
|
|
3496
|
+
state.nextDriftCheckAt = Math.max(1, ordinals.length) * DRIFT_CHECK_GROWTH_FACTOR;
|
|
3497
|
+
}
|
|
3498
|
+
function fieldHandlesOf(state, graph, searchable) {
|
|
3499
|
+
return {
|
|
3500
|
+
dimension: state.dimension,
|
|
3501
|
+
quantization: state.osq === null ? "none" : state.quantizationMode,
|
|
3502
|
+
metric: state.metric,
|
|
3503
|
+
store: state.store.handles,
|
|
3504
|
+
graph: graph === null ? null : graph.handles,
|
|
3505
|
+
filterThreshold: state.filterThreshold,
|
|
3506
|
+
searchable
|
|
3507
|
+
};
|
|
3508
|
+
}
|
|
3509
|
+
|
|
3510
|
+
// src/vector/hnsw/numbered-graph.ts
|
|
3511
|
+
var NO_NUMBER = -1;
|
|
3512
|
+
var NO_ORDINAL = -1;
|
|
3513
|
+
var NO_LAYER = -1;
|
|
3514
|
+
var BYTES_PER_VALUE = 4;
|
|
3515
|
+
var LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
3516
|
+
function swapEveryValue(bytes) {
|
|
3517
|
+
for (let i = 0; i + 3 < bytes.length; i += BYTES_PER_VALUE) {
|
|
3518
|
+
const first = bytes[i];
|
|
3519
|
+
const second = bytes[i + 1];
|
|
3520
|
+
bytes[i] = bytes[i + 3];
|
|
3521
|
+
bytes[i + 1] = bytes[i + 2];
|
|
3522
|
+
bytes[i + 2] = second;
|
|
3523
|
+
bytes[i + 3] = first;
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
function littleEndianBytesOf(values) {
|
|
3527
|
+
const bytes = new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
3528
|
+
if (!LITTLE_ENDIAN) swapEveryValue(bytes);
|
|
3529
|
+
return bytes;
|
|
3530
|
+
}
|
|
3531
|
+
function valuesOf(bytes) {
|
|
3532
|
+
if (LITTLE_ENDIAN && bytes.byteOffset % BYTES_PER_VALUE === 0) {
|
|
3533
|
+
return new Uint32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / BYTES_PER_VALUE));
|
|
3534
|
+
}
|
|
3535
|
+
const copy = bytes.slice(0, bytes.byteLength - bytes.byteLength % BYTES_PER_VALUE);
|
|
3536
|
+
if (!LITTLE_ENDIAN) swapEveryValue(copy);
|
|
3537
|
+
return new Uint32Array(copy.buffer, 0, copy.byteLength / BYTES_PER_VALUE);
|
|
3538
|
+
}
|
|
3539
|
+
function savedTopLayer(state, ordinal) {
|
|
3540
|
+
if (ordinal === NO_ORDINAL) return NO_LAYER;
|
|
3541
|
+
return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
|
|
3542
|
+
}
|
|
3543
|
+
function neighbourValueCount(state, topLayers, start, end) {
|
|
3544
|
+
let values = 0;
|
|
3545
|
+
for (let number = start; number < end; number++) {
|
|
3546
|
+
const top = topLayers[number];
|
|
3547
|
+
if (top < 0) continue;
|
|
3548
|
+
values += state.adjacency.level0Stride + top * state.adjacency.upperStride;
|
|
3549
|
+
}
|
|
3550
|
+
return values;
|
|
3551
|
+
}
|
|
3552
|
+
function serializePart(state, numbers, start, end) {
|
|
3553
|
+
const { numberOfOrdinal, ordinalOfNumber, topLayers } = numbers;
|
|
3554
|
+
const adjacency = state.adjacency;
|
|
3555
|
+
const levels = new Uint8Array(end - start);
|
|
3556
|
+
const values = new Uint32Array(neighbourValueCount(state, topLayers, start, end));
|
|
3557
|
+
let cursor = 0;
|
|
3558
|
+
for (let number = start; number < end; number++) {
|
|
3559
|
+
const top = topLayers[number];
|
|
3560
|
+
levels[number - start] = top + 1;
|
|
3561
|
+
for (let layer = 0; layer <= top; layer++) {
|
|
3562
|
+
const base = layerBase(adjacency, ordinalOfNumber[number], layer);
|
|
3563
|
+
const array = layerArray(adjacency, layer);
|
|
3564
|
+
const held = base === -1 ? 0 : Math.min(array[base], maxConns(state, layer));
|
|
3565
|
+
const countAt = cursor;
|
|
3566
|
+
cursor += 1;
|
|
3567
|
+
for (let i = 1; i <= held; i++) {
|
|
3568
|
+
const neighbourOrdinal = array[base + i];
|
|
3569
|
+
if (neighbourOrdinal < 0 || neighbourOrdinal >= numberOfOrdinal.length) continue;
|
|
3570
|
+
const neighbour = numberOfOrdinal[neighbourOrdinal];
|
|
3571
|
+
if (neighbour === NO_NUMBER || neighbour === number || topLayers[neighbour] < layer) continue;
|
|
3572
|
+
values[cursor] = neighbour;
|
|
3573
|
+
cursor += 1;
|
|
3574
|
+
}
|
|
3575
|
+
values[countAt] = cursor - countAt - 1;
|
|
3576
|
+
}
|
|
3577
|
+
}
|
|
3578
|
+
return { levels, neighbours: littleEndianBytesOf(values.subarray(0, cursor)) };
|
|
3579
|
+
}
|
|
3580
|
+
function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize) {
|
|
3581
|
+
const slots = adjacencySlots(state.adjacency);
|
|
3582
|
+
const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
|
|
3583
|
+
for (let number = 0; number < ordinalOfNumber.length; number++) {
|
|
3584
|
+
const ordinal = ordinalOfNumber[number];
|
|
3585
|
+
if (ordinal >= slots) continue;
|
|
3586
|
+
topLayers[number] = savedTopLayer(state, ordinal);
|
|
3587
|
+
}
|
|
3588
|
+
const entryOrdinal = entryPointOf(state);
|
|
3589
|
+
const entryNumber = entryOrdinal >= 0 && entryOrdinal < numberOfOrdinal.length ? numberOfOrdinal[entryOrdinal] : NO_NUMBER;
|
|
3590
|
+
const entryHoldsNode = entryNumber !== NO_NUMBER && topLayers[entryNumber] >= 0;
|
|
3591
|
+
const header = {
|
|
3592
|
+
entryPoint: entryHoldsNode ? entryNumber : null,
|
|
3593
|
+
maxLayer: Math.max(0, Math.min(topLayerOf(state), MAX_LAYER_CAP)),
|
|
3594
|
+
m: state.M,
|
|
3595
|
+
efConstruction: state.efCons,
|
|
3596
|
+
metric: state.buildMetric
|
|
3597
|
+
};
|
|
3598
|
+
const numbers = { numberOfOrdinal, ordinalOfNumber, topLayers };
|
|
3599
|
+
const parts = [];
|
|
3600
|
+
for (let start = 0; start < ordinalOfNumber.length; start += partSize) {
|
|
3601
|
+
const end = Math.min(start + partSize, ordinalOfNumber.length);
|
|
3602
|
+
parts.push({ ...header, ...serializePart(state, numbers, start, end) });
|
|
3603
|
+
}
|
|
3604
|
+
return parts;
|
|
3605
|
+
}
|
|
3606
|
+
function graphWithoutNodes(part, firstNumber, numbers) {
|
|
3607
|
+
const values = valuesOf(part.neighbours);
|
|
3608
|
+
const levels = part.levels.slice();
|
|
3609
|
+
const kept = new Uint32Array(values.length);
|
|
3610
|
+
let cursor = 0;
|
|
3611
|
+
let length = 0;
|
|
3612
|
+
for (let at = 0; at < part.levels.length; at++) {
|
|
3613
|
+
const start = cursor;
|
|
3614
|
+
for (let layer = 0; layer < part.levels[at]; layer++) cursor += values[cursor] + 1;
|
|
3615
|
+
if (numbers.has(firstNumber + at)) {
|
|
3616
|
+
levels[at] = 0;
|
|
3617
|
+
continue;
|
|
3618
|
+
}
|
|
3619
|
+
kept.set(values.subarray(start, cursor), length);
|
|
3620
|
+
length += cursor - start;
|
|
3621
|
+
}
|
|
3622
|
+
const entryPoint = part.entryPoint !== null && numbers.has(part.entryPoint) ? null : part.entryPoint;
|
|
3623
|
+
return { ...part, entryPoint, levels, neighbours: littleEndianBytesOf(kept.subarray(0, length)) };
|
|
3624
|
+
}
|
|
3625
|
+
function endsInsideAList() {
|
|
3626
|
+
throw new NarsilError(
|
|
3627
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3628
|
+
"Invalid vector graph payload: neighbours ends inside a list"
|
|
3629
|
+
);
|
|
3630
|
+
}
|
|
3631
|
+
function requireWholeNeighbourLists(graph) {
|
|
3632
|
+
const values = valuesOf(graph.neighbours);
|
|
3633
|
+
if (values.length * BYTES_PER_VALUE !== graph.neighbours.byteLength) endsInsideAList();
|
|
3634
|
+
let cursor = 0;
|
|
3635
|
+
for (let number = 0; number < graph.levels.length; number++) {
|
|
3636
|
+
for (let layer = 0; layer < graph.levels[number]; layer++) {
|
|
3637
|
+
if (cursor >= values.length) endsInsideAList();
|
|
3638
|
+
const count = values[cursor];
|
|
3639
|
+
if (count > values.length - cursor - 1) endsInsideAList();
|
|
3640
|
+
cursor += count + 1;
|
|
3641
|
+
}
|
|
3642
|
+
}
|
|
3643
|
+
}
|
|
3644
|
+
function topLayerInParts(graph, number) {
|
|
3645
|
+
const part = graph.parts[Math.floor(number / graph.partSize)];
|
|
3646
|
+
if (part === void 0) return -1;
|
|
3647
|
+
const at = number % graph.partSize;
|
|
3648
|
+
return at < part.levels.length ? part.levels[at] - 1 : -1;
|
|
3649
|
+
}
|
|
3650
|
+
function highestNodeOrdinal(graph, ordinalAt2) {
|
|
3651
|
+
let highest = -1;
|
|
3652
|
+
for (let p = 0; p < graph.parts.length; p++) {
|
|
3653
|
+
const levels = graph.parts[p].levels;
|
|
3654
|
+
for (let at = 0; at < levels.length; at++) {
|
|
3655
|
+
const ordinal = ordinalAt2(p * graph.partSize + at);
|
|
3656
|
+
if (levels[at] > 0 && ordinal > highest) highest = ordinal;
|
|
3657
|
+
}
|
|
3658
|
+
}
|
|
3659
|
+
return highest;
|
|
3660
|
+
}
|
|
3661
|
+
function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
|
|
3662
|
+
for (const part of graph.parts) requireWholeNeighbourLists(part);
|
|
3663
|
+
resetGraph(state);
|
|
3664
|
+
const ordinalAt2 = (number) => number < ordinalOfNumber.length ? ordinalOfNumber[number] : NO_ORDINAL;
|
|
3665
|
+
const highestOrdinal = highestNodeOrdinal(graph, ordinalAt2);
|
|
3666
|
+
if (highestOrdinal >= 0) ensureCapacity(state, highestOrdinal + 1);
|
|
3667
|
+
const kept = [];
|
|
3668
|
+
let highestLayer = -1;
|
|
3669
|
+
let nodeOnHighestLayer = NO_ORDINAL;
|
|
3670
|
+
for (let p = 0; p < graph.parts.length; p++) {
|
|
3671
|
+
const part = graph.parts[p];
|
|
3672
|
+
const values = valuesOf(part.neighbours);
|
|
3673
|
+
let cursor = 0;
|
|
3674
|
+
for (let at = 0; at < part.levels.length; at++) {
|
|
3675
|
+
const number = p * graph.partSize + at;
|
|
3676
|
+
const top = part.levels[at] - 1;
|
|
3677
|
+
if (top < 0) continue;
|
|
3678
|
+
const ordinal = ordinalAt2(number);
|
|
3679
|
+
const keptTop = Math.min(top, MAX_LAYER_CAP);
|
|
3680
|
+
if (ordinal !== NO_ORDINAL) {
|
|
3681
|
+
createNode(state.adjacency, ordinal, keptTop);
|
|
3682
|
+
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
3683
|
+
if (keptTop > highestLayer) {
|
|
3684
|
+
highestLayer = keptTop;
|
|
3685
|
+
nodeOnHighestLayer = ordinal;
|
|
3686
|
+
}
|
|
3687
|
+
}
|
|
3688
|
+
for (let layer = 0; layer <= top; layer++) {
|
|
3689
|
+
const count = values[cursor];
|
|
3690
|
+
kept.length = 0;
|
|
3691
|
+
const limit = maxConns(state, layer);
|
|
3692
|
+
for (let i = 1; i <= count && kept.length < limit; i++) {
|
|
3693
|
+
const neighbour = values[cursor + i];
|
|
3694
|
+
const neighbourOrdinal = ordinalAt2(neighbour);
|
|
3695
|
+
if (neighbourOrdinal === NO_ORDINAL || neighbour === number) continue;
|
|
3696
|
+
if (topLayerInParts(graph, neighbour) < layer) continue;
|
|
3697
|
+
if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
|
|
3698
|
+
}
|
|
3699
|
+
cursor += count + 1;
|
|
3700
|
+
if (ordinal !== NO_ORDINAL && layer <= keptTop) {
|
|
3701
|
+
writeNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
|
|
3702
|
+
}
|
|
3703
|
+
}
|
|
3704
|
+
}
|
|
3705
|
+
}
|
|
3706
|
+
const first = graph.parts[0];
|
|
3707
|
+
const entryPoint = first === void 0 ? null : first.entryPoint;
|
|
3708
|
+
const entryOrdinal = entryPoint === null ? NO_ORDINAL : ordinalAt2(entryPoint);
|
|
3709
|
+
const entryTop = entryPoint === null ? -1 : Math.min(topLayerInParts(graph, entryPoint), MAX_LAYER_CAP);
|
|
3710
|
+
const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
|
|
3711
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
|
|
3712
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
|
|
3713
|
+
leaveIncomingUncounted(state);
|
|
3714
|
+
}
|
|
3715
|
+
|
|
3716
|
+
// src/vector/hnsw/persistence.ts
|
|
3717
|
+
function resolveNodes(state, store, data) {
|
|
3718
|
+
const nodes = [];
|
|
3719
|
+
let maxOrd = -1;
|
|
3720
|
+
for (const [docId, maxLayer, layerConns] of data.nodes) {
|
|
3721
|
+
const ord = store.getOrdinal(docId);
|
|
3722
|
+
if (ord === void 0 || !store.holdsOrdinal(ord)) continue;
|
|
3723
|
+
const clampedMaxLayer = Math.min(Math.max(maxLayer, 0), MAX_LAYER_CAP);
|
|
3724
|
+
const layers = [];
|
|
3725
|
+
for (const [layer, neighbors] of layerConns) {
|
|
3726
|
+
if (layer < 0 || layer > clampedMaxLayer) continue;
|
|
3727
|
+
const limit = maxConns(state, layer);
|
|
3728
|
+
const resolved = [];
|
|
3729
|
+
for (const neighborDoc of neighbors) {
|
|
3730
|
+
if (resolved.length >= limit) break;
|
|
3731
|
+
const neighborOrd = store.getOrdinal(neighborDoc);
|
|
3732
|
+
if (neighborOrd === void 0) continue;
|
|
3733
|
+
addConnection(resolved, neighborOrd);
|
|
3734
|
+
if (neighborOrd > maxOrd) maxOrd = neighborOrd;
|
|
3735
|
+
}
|
|
3736
|
+
layers.push([layer, resolved]);
|
|
3737
|
+
}
|
|
3738
|
+
if (ord > maxOrd) maxOrd = ord;
|
|
3739
|
+
nodes.push({ ord, maxLayer: clampedMaxLayer, layers });
|
|
3740
|
+
}
|
|
3741
|
+
return { nodes, maxOrd };
|
|
3742
|
+
}
|
|
3743
|
+
function deserializeGraph(state, store, data) {
|
|
3744
|
+
resetGraph(state);
|
|
3745
|
+
const { nodes, maxOrd } = resolveNodes(state, store, data);
|
|
3746
|
+
if (maxOrd >= 0) {
|
|
3747
|
+
ensureCapacity(state, maxOrd + 1);
|
|
3748
|
+
}
|
|
3749
|
+
for (const node of nodes) {
|
|
3750
|
+
createNode(state.adjacency, node.ord, node.maxLayer);
|
|
3751
|
+
for (const [layer, neighbors] of node.layers) {
|
|
3752
|
+
writeNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
|
|
3753
|
+
}
|
|
3754
|
+
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
3755
|
+
}
|
|
3756
|
+
if (data.entryPoint != null && data.entryPoint !== "") {
|
|
3757
|
+
const epOrd = store.getOrdinal(data.entryPoint);
|
|
3758
|
+
if (epOrd !== void 0 && nodeExists(state, epOrd)) {
|
|
3759
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, epOrd);
|
|
3760
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, Math.min(Math.max(data.maxLayer, 0), MAX_LAYER_CAP));
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
if (entryPointOf(state) === -1 && Atomics.load(state.header, GRAPH_NODE_COUNT) > 0) {
|
|
3764
|
+
let bestOrd = -1;
|
|
3765
|
+
let bestLayer = -1;
|
|
3766
|
+
const slots = adjacencySlots(state.adjacency);
|
|
3767
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
3768
|
+
const level2 = nodeMaxLayer(state, ord);
|
|
3769
|
+
if (level2 === -1) continue;
|
|
3770
|
+
if (level2 > bestLayer) {
|
|
3771
|
+
bestLayer = level2;
|
|
3772
|
+
bestOrd = ord;
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, bestOrd);
|
|
3776
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, bestLayer);
|
|
3777
|
+
}
|
|
3778
|
+
leaveIncomingUncounted(state);
|
|
3779
|
+
}
|
|
3780
|
+
|
|
3781
|
+
// src/vector/hnsw/scratch-bytes.ts
|
|
3782
|
+
function listBytes(list) {
|
|
3783
|
+
return list.ords.byteLength + list.distances.byteLength;
|
|
3784
|
+
}
|
|
3785
|
+
function workspaceBytes(workspace) {
|
|
3786
|
+
let bytes = listBytes(workspace.frontier) + listBytes(workspace.found) + listBytes(workspace.traversal) + listBytes(workspace.working) + listBytes(workspace.pruneCandidates) + listBytes(workspace.pruneSelection) + listBytes(workspace.repairCandidates) + listBytes(workspace.repairSelection) + workspace.entryPoints.byteLength;
|
|
3787
|
+
for (const list of workspace.linkSelections) bytes += listBytes(list);
|
|
3788
|
+
return bytes;
|
|
3789
|
+
}
|
|
3790
|
+
function searchScratchBytes(state) {
|
|
3791
|
+
const held = workspaceBytes(state.workspace) + state.neighborScratch.byteLength + state.visited.byteLength;
|
|
3792
|
+
const native = attachedNativeField(state);
|
|
3793
|
+
return native === null ? held : held + nativeScratchBytes(state, native);
|
|
3794
|
+
}
|
|
3795
|
+
|
|
3796
|
+
// src/vector/hnsw/snapshot.ts
|
|
3797
|
+
function exportSnapshot(state) {
|
|
3798
|
+
const adj = state.adjacency;
|
|
3799
|
+
const slots = adjacencySlots(adj);
|
|
3800
|
+
const used = upperUsed(adj);
|
|
3801
|
+
const stride = adj.level0Stride;
|
|
3802
|
+
rebindNodes(adj);
|
|
3803
|
+
reachTombstone(state, slots);
|
|
3804
|
+
return {
|
|
3805
|
+
dimension: state.dimension,
|
|
3806
|
+
m: state.M,
|
|
3807
|
+
efConstruction: state.efCons,
|
|
3808
|
+
metric: state.buildMetric,
|
|
3809
|
+
slots,
|
|
3810
|
+
upperUsed: used,
|
|
3811
|
+
nodeLevels: adj.nodeLevels.slice(0, slots),
|
|
3812
|
+
level0: adj.level0.slice(0, slots * stride),
|
|
3813
|
+
upperBase: adj.upperBase.slice(0, slots),
|
|
3814
|
+
upper: adj.upper.slice(0, used),
|
|
3815
|
+
tombstones: state.tombstones.slice(0, slots),
|
|
3816
|
+
tombstoneCount: Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT),
|
|
3817
|
+
nodeCount: Atomics.load(state.header, GRAPH_NODE_COUNT),
|
|
3818
|
+
entryPointOrd: Atomics.load(state.header, GRAPH_ENTRY_POINT),
|
|
3819
|
+
topLayer: Atomics.load(state.header, GRAPH_TOP_LAYER)
|
|
3820
|
+
};
|
|
3821
|
+
}
|
|
3822
|
+
function restoreSnapshot(state, snapshot) {
|
|
3823
|
+
resetGraph(state);
|
|
3824
|
+
const adj = state.adjacency;
|
|
3825
|
+
ensureAdjacencyCapacity(adj, snapshot.slots);
|
|
3826
|
+
adj.nodeLevels.set(snapshot.nodeLevels);
|
|
3827
|
+
adj.level0.set(snapshot.level0);
|
|
3828
|
+
adj.upperBase.set(snapshot.upperBase);
|
|
3829
|
+
ensureUpperCapacity(adj, snapshot.upper.length);
|
|
3830
|
+
adj.upper.set(snapshot.upper);
|
|
3831
|
+
Atomics.store(state.header, GRAPH_UPPER_USED, snapshot.upperUsed);
|
|
3832
|
+
Atomics.store(state.header, GRAPH_SLOTS, snapshot.slots);
|
|
3833
|
+
state.tombstones = fixedView(adj.handles.tombstones, Uint8Array);
|
|
3834
|
+
state.tombstones.set(snapshot.tombstones);
|
|
3835
|
+
Atomics.store(state.header, GRAPH_TOMBSTONE_COUNT, snapshot.tombstoneCount);
|
|
3836
|
+
Atomics.store(state.header, GRAPH_NODE_COUNT, snapshot.nodeCount);
|
|
3837
|
+
Atomics.store(state.header, GRAPH_ENTRY_POINT, snapshot.entryPointOrd);
|
|
3838
|
+
Atomics.store(state.header, GRAPH_TOP_LAYER, snapshot.topLayer);
|
|
3839
|
+
leaveIncomingUncounted(state);
|
|
3840
|
+
}
|
|
3841
|
+
|
|
3842
|
+
// src/vector/hnsw/index.ts
|
|
3843
|
+
function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
3844
|
+
const M = config?.m ?? 16;
|
|
3845
|
+
const shared = handles ?? createSharedGraphHandles({
|
|
3846
|
+
m: M,
|
|
3847
|
+
mMax0: M * 2,
|
|
3848
|
+
efConstruction: config?.efConstruction ?? 200,
|
|
3849
|
+
metric: config?.metric ?? "cosine"
|
|
3850
|
+
});
|
|
3851
|
+
const state = openGraphState(shared, dimension, store, quantizer, 0);
|
|
3852
|
+
const docIdOf = (ord) => store.docIdForOrdinal(ord);
|
|
3853
|
+
function exclusively(operation) {
|
|
3854
|
+
lockGraphExclusive(state.locks);
|
|
3855
|
+
try {
|
|
3856
|
+
return operation();
|
|
3857
|
+
} finally {
|
|
3858
|
+
unlockGraphExclusive(state.locks);
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
function insertNode2(docId) {
|
|
3862
|
+
const ord = store.getOrdinal(docId);
|
|
3863
|
+
if (ord === void 0) {
|
|
3864
|
+
throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
|
|
3865
|
+
}
|
|
3866
|
+
const entry = store.entryForOrdinal(ord);
|
|
3867
|
+
if (!entry) {
|
|
3868
|
+
throw new Error(`Cannot insert HNSW node: vector for "${docId}" not found in VectorStore`);
|
|
3869
|
+
}
|
|
3870
|
+
if (entry.vector.length !== dimension) {
|
|
3871
|
+
throw new NarsilError(
|
|
3872
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
3873
|
+
`Vector dimension mismatch: expected ${dimension}, got ${entry.vector.length}`,
|
|
3874
|
+
{ docId, expected: dimension, received: entry.vector.length }
|
|
3875
|
+
);
|
|
3876
|
+
}
|
|
3877
|
+
insertNode(state, ord);
|
|
3878
|
+
}
|
|
3879
|
+
function* entriesIterator() {
|
|
3880
|
+
const slots = adjacencySlots(state.adjacency);
|
|
3881
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
3882
|
+
if (!hasNode(state.adjacency, ord)) continue;
|
|
3883
|
+
if (isTombstoned(state, ord)) continue;
|
|
3884
|
+
const docId = store.docIdForOrdinal(ord);
|
|
3885
|
+
if (docId === void 0) continue;
|
|
3886
|
+
const entry = store.entryForOrdinal(ord);
|
|
3887
|
+
if (!entry) continue;
|
|
3888
|
+
yield [docId, { docId, vector: entry.vector, magnitude: entry.magnitude }];
|
|
3889
|
+
}
|
|
3890
|
+
}
|
|
3891
|
+
function compactionNeeded() {
|
|
3892
|
+
const nodeCount = nodeCountOf(state);
|
|
3893
|
+
if (nodeCount === 0) return false;
|
|
3894
|
+
const tombstones = tombstoneCountOf(state);
|
|
3895
|
+
return tombstones / nodeCount > COMPACTION_TOMBSTONE_RATIO || tombstones > COMPACTION_ABSOLUTE_THRESHOLD;
|
|
3896
|
+
}
|
|
3897
|
+
return {
|
|
3898
|
+
get dimension() {
|
|
3899
|
+
return dimension;
|
|
3900
|
+
},
|
|
3901
|
+
get size() {
|
|
3902
|
+
return nodeCountOf(state) - tombstoneCountOf(state);
|
|
3903
|
+
},
|
|
3904
|
+
get tombstoneCount() {
|
|
3905
|
+
return tombstoneCountOf(state);
|
|
3906
|
+
},
|
|
3907
|
+
get entryPointId() {
|
|
3908
|
+
const entryPoint = entryPointOf(state);
|
|
3909
|
+
return entryPoint === -1 ? null : store.docIdForOrdinal(entryPoint) ?? null;
|
|
3910
|
+
},
|
|
3911
|
+
get topLayer() {
|
|
3912
|
+
return topLayerOf(state);
|
|
3913
|
+
},
|
|
3914
|
+
get m() {
|
|
3915
|
+
return state.M;
|
|
3916
|
+
},
|
|
3917
|
+
get efConstruction() {
|
|
3918
|
+
return state.efCons;
|
|
3919
|
+
},
|
|
3920
|
+
get metric() {
|
|
3921
|
+
return state.buildMetric;
|
|
3922
|
+
},
|
|
3923
|
+
get graphBytes() {
|
|
3924
|
+
return graphBytes(state.adjacency);
|
|
3925
|
+
},
|
|
3926
|
+
get searchScratchBytes() {
|
|
3927
|
+
return searchScratchBytes(state);
|
|
3928
|
+
},
|
|
3929
|
+
get handles() {
|
|
3930
|
+
return shared;
|
|
3931
|
+
},
|
|
3932
|
+
insertNode: insertNode2,
|
|
3933
|
+
insertOrdinal: (ordinal) => insertNode(state, ordinal),
|
|
3934
|
+
markTombstone: (docId) => {
|
|
3935
|
+
const ord = store.getOrdinal(docId);
|
|
3936
|
+
if (ord !== void 0) markTombstone(state, ord);
|
|
3937
|
+
},
|
|
3938
|
+
markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
|
|
3939
|
+
holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
|
|
3940
|
+
holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
|
|
3941
|
+
has: (docId) => {
|
|
3942
|
+
const ord = store.getOrdinal(docId);
|
|
3943
|
+
return ord !== void 0 && hasNode(state.adjacency, ord) && !isTombstoned(state, ord);
|
|
3944
|
+
},
|
|
3945
|
+
isTombstoned: (docId) => {
|
|
3946
|
+
const ord = store.getOrdinal(docId);
|
|
3947
|
+
return ord !== void 0 && isTombstoned(state, ord);
|
|
3948
|
+
},
|
|
3949
|
+
search: (query, k, searchMetric, minSimilarity, options) => search(state, docIdOf, query, k, searchMetric, minSimilarity, options),
|
|
3950
|
+
clear: () => exclusively(() => resetGraph(state)),
|
|
3951
|
+
retire: () => Atomics.store(shared.header, GRAPH_RETIRED, 1),
|
|
3952
|
+
entries: entriesIterator,
|
|
3953
|
+
compactionNeeded,
|
|
3954
|
+
compactTombstones: () => exclusively(() => compactTombstones(state)),
|
|
3955
|
+
connectUnreached: () => exclusively(() => connectAfterBuild(state)),
|
|
3956
|
+
rebuild: () => exclusively(() => rebuild(state)),
|
|
3957
|
+
exclusively,
|
|
3958
|
+
deserialize: (data) => exclusively(() => deserializeGraph(state, store, data)),
|
|
3959
|
+
serializeNumbered: (numberOfOrdinal, ordinalOfNumber, partSize) => serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize),
|
|
3960
|
+
deserializeNumbered: (graph, ordinalOfNumber) => exclusively(() => deserializeNumberedGraph(state, graph, ordinalOfNumber)),
|
|
3961
|
+
exportSnapshot: () => exportSnapshot(state),
|
|
3962
|
+
restoreSnapshot: (snapshot) => exclusively(() => restoreSnapshot(state, snapshot))
|
|
3963
|
+
};
|
|
3964
|
+
}
|
|
3965
|
+
|
|
3966
|
+
// src/vector/vector-index/persistence.ts
|
|
3967
|
+
function recordFor(state, quantizer, docId) {
|
|
3968
|
+
const record = quantizer.recordOf(docId);
|
|
3969
|
+
if (record !== void 0) return record;
|
|
3970
|
+
if (!state.store.has(docId)) return void 0;
|
|
3971
|
+
quantizer.quantize(docId);
|
|
3972
|
+
return quantizer.recordOf(docId);
|
|
3973
|
+
}
|
|
3974
|
+
function codedCentroid(state) {
|
|
3975
|
+
const centroid = state.osq?.centroid ?? null;
|
|
3976
|
+
if (centroid === null || state.store.handles.codeLayout === null || state.hnsw === null) return null;
|
|
3977
|
+
return Array.from(centroid);
|
|
3978
|
+
}
|
|
3979
|
+
function sameCentroid(state, planned) {
|
|
3980
|
+
const current = state.osq?.centroid ?? null;
|
|
3981
|
+
if (current === null || current.length !== planned.length) return false;
|
|
3982
|
+
for (let i = 0; i < planned.length; i++) {
|
|
3983
|
+
if (current[i] !== planned[i]) return false;
|
|
3984
|
+
}
|
|
3985
|
+
return true;
|
|
3986
|
+
}
|
|
3987
|
+
function codesAgainstPlannedCentroid(quantizer, slotStride, centroid, vectors, ordinals) {
|
|
3988
|
+
const planned = Float32Array.from(centroid);
|
|
3989
|
+
const dimension = planned.length;
|
|
3990
|
+
const scratch = createOsqScratch(dimension);
|
|
3991
|
+
const records = new Uint8Array(ordinals.length * slotStride);
|
|
3992
|
+
const trailer = new DataView(records.buffer, records.byteOffset, records.byteLength);
|
|
3993
|
+
for (let i = 0; i < ordinals.length; i++) {
|
|
3994
|
+
if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
|
|
3995
|
+
const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
|
|
3996
|
+
const code = osqQuantize(vector, planned, quantizer.bits, quantizer.metric, scratch);
|
|
3997
|
+
writeRecord(records, i * slotStride, code, quantizer.bits, trailer);
|
|
3998
|
+
}
|
|
3999
|
+
return { bits: quantizer.bits, centroid, records };
|
|
4000
|
+
}
|
|
4001
|
+
function partCodes(state, docIds, centroid, tombstonedAt, vectors, ordinals) {
|
|
4002
|
+
const quantizer = state.osq;
|
|
4003
|
+
const layout = state.store.handles.codeLayout;
|
|
4004
|
+
if (centroid === null || quantizer === null || layout === null) return null;
|
|
4005
|
+
if (!sameCentroid(state, centroid)) {
|
|
4006
|
+
return codesAgainstPlannedCentroid(quantizer, layout.slotStride, centroid, vectors, ordinals);
|
|
4007
|
+
}
|
|
4008
|
+
const records = new Uint8Array(docIds.length * layout.slotStride);
|
|
4009
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
4010
|
+
const tombstoned = tombstonedAt.get(i);
|
|
4011
|
+
const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
|
|
4012
|
+
if (record !== void 0) records.set(record, i * layout.slotStride);
|
|
4013
|
+
}
|
|
4014
|
+
return { bits: quantizer.bits, centroid, records };
|
|
4015
|
+
}
|
|
4016
|
+
function splitSequences(parts) {
|
|
4017
|
+
const sequences = [];
|
|
4018
|
+
let current = null;
|
|
4019
|
+
for (const part of parts) {
|
|
4020
|
+
if (current === null || part.part === 0) {
|
|
4021
|
+
current = { docIds: [], graphs: [], codes: null, codesComplete: true };
|
|
4022
|
+
sequences.push(current);
|
|
4023
|
+
}
|
|
4024
|
+
current.docIds.push(...part.docIds);
|
|
4025
|
+
for (let index = 0; index < part.graphs.length; index++) {
|
|
4026
|
+
const graph = part.graphs[index];
|
|
4027
|
+
const held = current.graphs[index];
|
|
4028
|
+
if (held === void 0) current.graphs[index] = { ...graph, nodes: [...graph.nodes] };
|
|
4029
|
+
else held.nodes.push(...graph.nodes);
|
|
4030
|
+
}
|
|
4031
|
+
if (part.codes === null) {
|
|
4032
|
+
if (part.docIds.length > 0) current.codesComplete = false;
|
|
4033
|
+
continue;
|
|
4034
|
+
}
|
|
4035
|
+
if (current.codes === null) {
|
|
4036
|
+
current.codes = { bits: part.codes.bits, centroid: part.codes.centroid, records: part.codes.records };
|
|
4037
|
+
continue;
|
|
4038
|
+
}
|
|
4039
|
+
if (current.codes.bits !== part.codes.bits) {
|
|
4040
|
+
current.codesComplete = false;
|
|
4041
|
+
continue;
|
|
4042
|
+
}
|
|
4043
|
+
const joined = new Uint8Array(current.codes.records.length + part.codes.records.length);
|
|
4044
|
+
joined.set(current.codes.records, 0);
|
|
4045
|
+
joined.set(part.codes.records, current.codes.records.length);
|
|
4046
|
+
current.codes = { ...current.codes, records: joined };
|
|
4047
|
+
}
|
|
4048
|
+
return sequences;
|
|
4049
|
+
}
|
|
4050
|
+
function restoreCodes(state, sequences) {
|
|
4051
|
+
const quantizer = state.osq;
|
|
4052
|
+
const layout = state.store.handles.codeLayout;
|
|
4053
|
+
if (quantizer === null || layout === null) return;
|
|
4054
|
+
const usable = sequences.filter((sequence) => sequence.docIds.length > 0);
|
|
4055
|
+
const single = usable.length === 1 ? usable[0] : null;
|
|
4056
|
+
if (single === null || single.codes === null || !single.codesComplete || single.codes.bits !== quantizer.bits) {
|
|
4057
|
+
recalibrateFromStore(state);
|
|
4058
|
+
return;
|
|
4059
|
+
}
|
|
4060
|
+
quantizer.restoreCentroid(Float32Array.from(single.codes.centroid));
|
|
4061
|
+
const recordBytes = layout.slotStride;
|
|
4062
|
+
for (let i = 0; i < single.docIds.length; i++) {
|
|
4063
|
+
quantizer.restoreRecord(single.docIds[i], single.codes.records.subarray(i * recordBytes, (i + 1) * recordBytes));
|
|
4064
|
+
}
|
|
4065
|
+
}
|
|
4066
|
+
function restoreGraphs(state, graphs) {
|
|
4067
|
+
if (graphs.length === 0) {
|
|
4068
|
+
for (const docId of state.store.docIds()) {
|
|
4069
|
+
if (!state.tombstones.has(docId)) state.buffer.add(docId);
|
|
4070
|
+
}
|
|
4071
|
+
return;
|
|
4072
|
+
}
|
|
4073
|
+
const graphData = graphs[0];
|
|
4074
|
+
const restoredHnsw = createHNSWIndex(
|
|
4075
|
+
state.dimension,
|
|
4076
|
+
state.store,
|
|
4077
|
+
{
|
|
4078
|
+
m: graphData.m ?? state.hnswConfig?.m,
|
|
4079
|
+
efConstruction: graphData.efConstruction ?? state.hnswConfig?.efConstruction,
|
|
4080
|
+
metric: graphData.metric ?? state.hnswConfig?.metric
|
|
4081
|
+
},
|
|
4082
|
+
state.osq ?? void 0
|
|
4083
|
+
);
|
|
4084
|
+
restoredHnsw.deserialize(graphData);
|
|
4085
|
+
adoptGraph(state, restoredHnsw);
|
|
4086
|
+
for (let i = 1; i < graphs.length; i++) {
|
|
4087
|
+
for (const [nodeDocId] of graphs[i].nodes) {
|
|
4088
|
+
if (!restoredHnsw.has(nodeDocId) && state.store.has(nodeDocId)) restoredHnsw.insertNode(nodeDocId);
|
|
4089
|
+
}
|
|
4090
|
+
}
|
|
4091
|
+
for (const docId of state.store.docIds()) {
|
|
4092
|
+
if (state.tombstones.has(docId)) continue;
|
|
4093
|
+
if (!restoredHnsw.has(docId)) state.buffer.add(docId);
|
|
4094
|
+
}
|
|
4095
|
+
if (state.buffer.size > restoredHnsw.size) {
|
|
4096
|
+
adoptGraph(state, null);
|
|
4097
|
+
state.buffer.clear();
|
|
4098
|
+
for (const docId of state.store.docIds()) {
|
|
4099
|
+
if (!state.tombstones.has(docId)) state.buffer.add(docId);
|
|
4100
|
+
}
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
function insertPart(state, part) {
|
|
4104
|
+
const dimension = state.dimension;
|
|
4105
|
+
const vectors = bytesToVectors(part.vectors);
|
|
4106
|
+
for (let i = 0; i < part.docIds.length; i++) {
|
|
4107
|
+
state.store.insert(part.docIds[i], vectors.subarray(i * dimension, (i + 1) * dimension));
|
|
4108
|
+
}
|
|
4109
|
+
}
|
|
4110
|
+
function deserialize(state, parts) {
|
|
4111
|
+
for (const part of parts) {
|
|
4112
|
+
if (part.dimension !== state.dimension) {
|
|
4113
|
+
throw new NarsilError(
|
|
4114
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
4115
|
+
`Payload dimension ${part.dimension} does not match index dimension ${state.dimension}`,
|
|
4116
|
+
{ expected: state.dimension, received: part.dimension }
|
|
4117
|
+
);
|
|
4118
|
+
}
|
|
4119
|
+
}
|
|
4120
|
+
const sequences = splitSequences(parts);
|
|
4121
|
+
const graphs = [];
|
|
4122
|
+
for (const sequence of sequences) {
|
|
4123
|
+
for (const graph of sequence.graphs) {
|
|
4124
|
+
if (graph.nodes.length > 0) graphs.push(graph);
|
|
4125
|
+
}
|
|
4126
|
+
}
|
|
4127
|
+
emptyFieldBeforeRestore(state);
|
|
4128
|
+
for (const part of parts) insertPart(state, part);
|
|
4129
|
+
if (state.osq !== null && osqBitsOf(state.quantizationMode) !== null && graphs.length > 0) {
|
|
4130
|
+
restoreCodes(state, sequences);
|
|
4131
|
+
}
|
|
4132
|
+
restoreGraphs(state, graphs);
|
|
4133
|
+
}
|
|
4134
|
+
|
|
4135
|
+
// src/vector/vector-index/checkpoint-plan.ts
|
|
4136
|
+
var ORDINAL_HOLDS_NO_NUMBER = -1;
|
|
4137
|
+
function vectorsPerFile(dimension, codeBits) {
|
|
4138
|
+
const recordBytes = codeBits === 1 || codeBits === 2 || codeBits === 4 || codeBits === 8 ? osqRecordBytes(dimension, codeBits) : 0;
|
|
4139
|
+
const docIdBytes = VECTOR_FILE_STRING_HEADER_BYTES + MAX_DOC_ID_LENGTH * 3;
|
|
4140
|
+
const perVector = dimension * 4 + recordBytes + docIdBytes;
|
|
4141
|
+
const overhead = VECTOR_FILE_FIXED_OVERHEAD_BYTES + dimension * VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT;
|
|
4142
|
+
const fitting = Math.floor((MAX_FILE_BYTES - overhead) / perVector);
|
|
4143
|
+
return Math.max(1, Math.min(VECTOR_FILE_MAX_VECTORS, fitting));
|
|
4144
|
+
}
|
|
4145
|
+
function fileCapacityOf(state) {
|
|
4146
|
+
return vectorsPerFile(state.dimension, osqBitsOf(state.quantizationMode));
|
|
4147
|
+
}
|
|
4148
|
+
function calibrationOf(state) {
|
|
4149
|
+
if (codedCentroid(state) === null) return null;
|
|
4150
|
+
return Atomics.load(state.store.handles.header, STORE_CALIBRATION_GENERATION);
|
|
4151
|
+
}
|
|
4152
|
+
function signatureOf(state) {
|
|
4153
|
+
return { revision: state.revision, graphSize: state.hnsw?.size ?? -1, calibration: calibrationOf(state) };
|
|
4154
|
+
}
|
|
4155
|
+
function sameSignature(a, b) {
|
|
4156
|
+
return a !== null && a.revision === b.revision && a.graphSize === b.graphSize && a.calibration === b.calibration;
|
|
4157
|
+
}
|
|
4158
|
+
function registryMatches(state, listedKeys) {
|
|
4159
|
+
const listed = listedKeys ?? [];
|
|
4160
|
+
if (listed.length !== state.savedFiles.length) return false;
|
|
4161
|
+
return state.savedFiles.every((file, index) => file.key === listed[index]);
|
|
4162
|
+
}
|
|
4163
|
+
function isLive2(state, ordinal) {
|
|
4164
|
+
if (ordinal === NO_LIVE_VECTOR_AT_POSITION) return false;
|
|
4165
|
+
const docId = state.store.docIdForOrdinal(ordinal);
|
|
4166
|
+
return docId !== void 0 && !state.tombstones.has(docId);
|
|
4167
|
+
}
|
|
4168
|
+
function isGraphTombstone(state, ordinal) {
|
|
4169
|
+
return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
|
|
4170
|
+
}
|
|
4171
|
+
function isCarried(state, ordinal) {
|
|
4172
|
+
return isLive2(state, ordinal) || isGraphTombstone(state, ordinal);
|
|
4173
|
+
}
|
|
4174
|
+
function carriedCountOf(state, file) {
|
|
4175
|
+
let carried = 0;
|
|
4176
|
+
for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
|
|
4177
|
+
return carried;
|
|
4178
|
+
}
|
|
4179
|
+
function deadBitsOf(state, file) {
|
|
4180
|
+
return deadBitsWhere(file.ordinals.length, (position) => !isLive2(state, file.ordinals[position]));
|
|
4181
|
+
}
|
|
4182
|
+
function filesWorthKeeping(state, listed, calibration, carriedCounts) {
|
|
4183
|
+
const kept = [];
|
|
4184
|
+
for (const file of listed) {
|
|
4185
|
+
const carried = carriedCountOf(state, file);
|
|
4186
|
+
const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
|
|
4187
|
+
if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
|
|
4188
|
+
kept.push(file);
|
|
4189
|
+
carriedCounts.push(carried);
|
|
4190
|
+
}
|
|
4191
|
+
return kept;
|
|
4192
|
+
}
|
|
4193
|
+
function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
|
|
4194
|
+
let pending = vectorsToWrite;
|
|
4195
|
+
for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
|
|
4196
|
+
if (kept[index].ordinals.length >= capacity) continue;
|
|
4197
|
+
if (carriedCounts[index] > pending) return;
|
|
4198
|
+
pending += carriedCounts[index];
|
|
4199
|
+
kept.splice(index, 1);
|
|
4200
|
+
carriedCounts.splice(index, 1);
|
|
4201
|
+
}
|
|
4202
|
+
}
|
|
4203
|
+
function ordinalsOutsideOf(state, kept) {
|
|
4204
|
+
const slots = state.store.slots;
|
|
4205
|
+
const saved = new Uint8Array(slots);
|
|
4206
|
+
for (const file of kept) {
|
|
4207
|
+
for (let i = 0; i < file.ordinals.length; i++) {
|
|
4208
|
+
const ordinal = file.ordinals[i];
|
|
4209
|
+
if (ordinal !== NO_LIVE_VECTOR_AT_POSITION && ordinal < slots) saved[ordinal] = 1;
|
|
4210
|
+
}
|
|
4211
|
+
}
|
|
4212
|
+
const outside = [];
|
|
4213
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
4214
|
+
if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
|
|
4215
|
+
}
|
|
4216
|
+
return outside;
|
|
4217
|
+
}
|
|
4218
|
+
function countOutside(state, carriedCounts) {
|
|
4219
|
+
let held = 0;
|
|
4220
|
+
for (const carried of carriedCounts) held += carried;
|
|
4221
|
+
const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
|
|
4222
|
+
return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
|
|
4223
|
+
}
|
|
4224
|
+
function numberEveryVector(state, kept, written) {
|
|
4225
|
+
let total = written.length;
|
|
4226
|
+
for (const file of kept) total += file.ordinals.length;
|
|
4227
|
+
const numberOfOrdinal = new Int32Array(state.store.slots).fill(ORDINAL_HOLDS_NO_NUMBER);
|
|
4228
|
+
const ordinalOfNumber = new Int32Array(total).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
4229
|
+
let number = 0;
|
|
4230
|
+
for (const file of kept) {
|
|
4231
|
+
for (let i = 0; i < file.ordinals.length; i++, number++) {
|
|
4232
|
+
const ordinal = file.ordinals[i];
|
|
4233
|
+
if (!isCarried(state, ordinal)) continue;
|
|
4234
|
+
numberOfOrdinal[ordinal] = number;
|
|
4235
|
+
ordinalOfNumber[number] = ordinal;
|
|
4236
|
+
}
|
|
4237
|
+
}
|
|
4238
|
+
for (const ordinal of written) {
|
|
4239
|
+
numberOfOrdinal[ordinal] = number;
|
|
4240
|
+
ordinalOfNumber[number] = ordinal;
|
|
4241
|
+
number += 1;
|
|
4242
|
+
}
|
|
4243
|
+
return { numberOfOrdinal, ordinalOfNumber };
|
|
4244
|
+
}
|
|
4245
|
+
function withBuildHistory(state, graph) {
|
|
4246
|
+
const shuffledBuildCount = state.shuffledBuildCount;
|
|
4247
|
+
if (shuffledBuildCount === null) return graph;
|
|
4248
|
+
const calibratedMeanLength = state.calibratedMeanLength;
|
|
4249
|
+
if (calibratedMeanLength === null) return { ...graph, shuffledBuildCount };
|
|
4250
|
+
return { ...graph, shuffledBuildCount, calibratedMeanLength };
|
|
4251
|
+
}
|
|
4252
|
+
function graphPartsOf(state, numbers) {
|
|
4253
|
+
if (state.hnsw === null || numbers.ordinalOfNumber.length === 0) return null;
|
|
4254
|
+
const parts = state.hnsw.serializeNumbered(
|
|
4255
|
+
numbers.numberOfOrdinal,
|
|
4256
|
+
numbers.ordinalOfNumber,
|
|
4257
|
+
VECTOR_GRAPH_PART_VECTORS
|
|
4258
|
+
);
|
|
4259
|
+
return parts.map((graph) => ({
|
|
4260
|
+
v: VECTOR_GRAPH_PART_PAYLOAD_VERSION,
|
|
4261
|
+
fieldName: state.fieldName,
|
|
4262
|
+
graphs: [withBuildHistory(state, graph)]
|
|
4263
|
+
}));
|
|
4264
|
+
}
|
|
4265
|
+
function graphPartsWithoutNodes(parts, numbers) {
|
|
4266
|
+
return parts.map((part, index) => ({
|
|
4267
|
+
...part,
|
|
4268
|
+
graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
|
|
4269
|
+
}));
|
|
4270
|
+
}
|
|
4271
|
+
function plannedVectorOf(state, ordinal) {
|
|
4272
|
+
const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
|
|
4273
|
+
return { ordinal, docId, tombstoned: !isLive2(state, ordinal) };
|
|
4274
|
+
}
|
|
4275
|
+
function readFile(state, planned, centroid) {
|
|
4276
|
+
const dimension = state.dimension;
|
|
4277
|
+
const vectors = new Float32Array(planned.length * dimension);
|
|
4278
|
+
const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
4279
|
+
const tombstonedAt = /* @__PURE__ */ new Map();
|
|
4280
|
+
const docIds = planned.map((vector) => vector.docId);
|
|
4281
|
+
for (let i = 0; i < planned.length; i++) {
|
|
4282
|
+
const { ordinal, docId, tombstoned } = planned[i];
|
|
4283
|
+
const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
|
|
4284
|
+
const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
|
|
4285
|
+
if (held === void 0 || entry === void 0) continue;
|
|
4286
|
+
vectors.set(entry.vector, i * dimension);
|
|
4287
|
+
ordinals[i] = held;
|
|
4288
|
+
if (tombstoned) tombstonedAt.set(i, held);
|
|
4289
|
+
}
|
|
4290
|
+
return {
|
|
4291
|
+
payload: {
|
|
4292
|
+
v: VECTOR_FILE_PAYLOAD_VERSION,
|
|
4293
|
+
fieldName: state.fieldName,
|
|
4294
|
+
dimension,
|
|
4295
|
+
docIds,
|
|
4296
|
+
codes: partCodes(state, docIds, centroid, tombstonedAt, vectors, ordinals),
|
|
4297
|
+
vectors: vectorsToBytes(vectors)
|
|
4298
|
+
},
|
|
4299
|
+
ordinals,
|
|
4300
|
+
dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
|
|
4301
|
+
};
|
|
4302
|
+
}
|
|
4303
|
+
function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
4304
|
+
const signature = signatureOf(state);
|
|
4305
|
+
const calibration = signature.calibration;
|
|
4306
|
+
const trusted = registryMatches(state, listedKeys);
|
|
4307
|
+
const capacity = fileCapacityOf(state);
|
|
4308
|
+
const carriedCounts = [];
|
|
4309
|
+
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
|
|
4310
|
+
absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
|
|
4311
|
+
const written = ordinalsOutsideOf(state, keptFiles);
|
|
4312
|
+
const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
|
|
4313
|
+
const centroid = codedCentroid(state);
|
|
4314
|
+
const listedBefore = trusted && listedKeys !== null;
|
|
4315
|
+
const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
|
|
4316
|
+
let firstWrittenNumber = 0;
|
|
4317
|
+
for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
|
|
4318
|
+
const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
|
|
4319
|
+
const vanished = /* @__PURE__ */ new Set();
|
|
4320
|
+
let vanishedWhenStripped = 0;
|
|
4321
|
+
let graphPartsToWrite = graphParts;
|
|
4322
|
+
return {
|
|
4323
|
+
unchanged,
|
|
4324
|
+
keptFiles,
|
|
4325
|
+
kept: keptFiles.map((file) => ({
|
|
4326
|
+
id: file.id,
|
|
4327
|
+
key: file.key,
|
|
4328
|
+
count: file.ordinals.length,
|
|
4329
|
+
dead: deadBitsOf(state, file)
|
|
4330
|
+
})),
|
|
4331
|
+
newFiles: Math.ceil(written.length / capacity),
|
|
4332
|
+
get graphParts() {
|
|
4333
|
+
if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
|
|
4334
|
+
graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
|
|
4335
|
+
vanishedWhenStripped = vanished.size;
|
|
4336
|
+
}
|
|
4337
|
+
return graphPartsToWrite;
|
|
4338
|
+
},
|
|
4339
|
+
signature,
|
|
4340
|
+
readNewFile(index) {
|
|
4341
|
+
const start = index * capacity;
|
|
4342
|
+
const end = start + capacity;
|
|
4343
|
+
const file = readFile(state, planned.slice(start, end), centroid);
|
|
4344
|
+
for (let i = 0; i < file.ordinals.length; i++) {
|
|
4345
|
+
if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
|
|
4346
|
+
}
|
|
4347
|
+
return file;
|
|
4348
|
+
}
|
|
4349
|
+
};
|
|
4350
|
+
}
|
|
4351
|
+
function recordCheckpoint(state, plan, written) {
|
|
4352
|
+
if (plan.unchanged) return;
|
|
4353
|
+
const calibration = plan.signature.calibration;
|
|
4354
|
+
state.savedFiles = [...plan.keptFiles, ...written.map((file) => ({ ...file, calibration }))];
|
|
4355
|
+
state.savedSignature = plan.signature;
|
|
4356
|
+
}
|
|
4357
|
+
|
|
4358
|
+
export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
|