@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -0,0 +1,745 @@
|
|
|
1
|
+
import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-MWUK3DA5.mjs';
|
|
2
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-MWUK3DA5.mjs';
|
|
3
|
+
import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, COLD_WALK_NODES_READ_AHEAD } from './chunk-MVM33ZUL.mjs';
|
|
4
|
+
import './chunk-CCOAOO73.mjs';
|
|
5
|
+
import './chunk-TCLEOZ4P.mjs';
|
|
6
|
+
import './chunk-WCFCHIBB.mjs';
|
|
7
|
+
import { magnitude, defaultRescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, scalarQueryProducts, osqEstimate, osqDistance, toDistance } from './chunk-HKX2G6WH.mjs';
|
|
8
|
+
export { createHNSWIndex, osqBitsOf } from './chunk-HKX2G6WH.mjs';
|
|
9
|
+
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, stageQueryLevels, readTrailer } from './chunk-LVY2LI35.mjs';
|
|
10
|
+
import './chunk-3ZBJYAS4.mjs';
|
|
11
|
+
import { HEADER_SIZE, crc32 } from './chunk-5DB2VDGI.mjs';
|
|
12
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
13
|
+
import './chunk-26A5JFT2.mjs';
|
|
14
|
+
import { compareCodePoints } from './chunk-D33A5VMU.mjs';
|
|
15
|
+
import './chunk-MZI3TEOW.mjs';
|
|
16
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
17
|
+
import { decode, decodeMulti } from '@msgpack/msgpack';
|
|
18
|
+
|
|
19
|
+
// src/persistence/paged/cold-search.ts
|
|
20
|
+
function pushMin(heap, number, distance) {
|
|
21
|
+
let at = heap.numbers.length;
|
|
22
|
+
heap.numbers.push(number);
|
|
23
|
+
heap.distances.push(distance);
|
|
24
|
+
while (at > 0) {
|
|
25
|
+
const parent = at - 1 >> 1;
|
|
26
|
+
if (heap.distances[parent] <= distance) break;
|
|
27
|
+
heap.numbers[at] = heap.numbers[parent];
|
|
28
|
+
heap.distances[at] = heap.distances[parent];
|
|
29
|
+
at = parent;
|
|
30
|
+
}
|
|
31
|
+
heap.numbers[at] = number;
|
|
32
|
+
heap.distances[at] = distance;
|
|
33
|
+
}
|
|
34
|
+
function popMin(heap) {
|
|
35
|
+
const size = heap.numbers.length;
|
|
36
|
+
if (size === 0) return null;
|
|
37
|
+
const top = { number: heap.numbers[0], distance: heap.distances[0] };
|
|
38
|
+
const number = heap.numbers[size - 1];
|
|
39
|
+
const distance = heap.distances[size - 1];
|
|
40
|
+
heap.numbers.pop();
|
|
41
|
+
heap.distances.pop();
|
|
42
|
+
const remaining = size - 1;
|
|
43
|
+
if (remaining === 0) return top;
|
|
44
|
+
let at = 0;
|
|
45
|
+
for (; ; ) {
|
|
46
|
+
let child = 2 * at + 1;
|
|
47
|
+
if (child >= remaining) break;
|
|
48
|
+
if (child + 1 < remaining && heap.distances[child + 1] < heap.distances[child]) child += 1;
|
|
49
|
+
if (heap.distances[child] >= distance) break;
|
|
50
|
+
heap.numbers[at] = heap.numbers[child];
|
|
51
|
+
heap.distances[at] = heap.distances[child];
|
|
52
|
+
at = child;
|
|
53
|
+
}
|
|
54
|
+
heap.numbers[at] = number;
|
|
55
|
+
heap.distances[at] = distance;
|
|
56
|
+
return top;
|
|
57
|
+
}
|
|
58
|
+
function negated(heap) {
|
|
59
|
+
return heap.distances.length === 0 ? Number.POSITIVE_INFINITY : -heap.distances[0];
|
|
60
|
+
}
|
|
61
|
+
function neighbourList(data, number, layer) {
|
|
62
|
+
const part = data.parts[Math.floor(number / COLD_GRAPH_PART_VECTORS)];
|
|
63
|
+
const offset = number % COLD_GRAPH_PART_VECTORS;
|
|
64
|
+
const start = part.nodeOffsets[offset];
|
|
65
|
+
const end = part.nodeOffsets[offset + 1];
|
|
66
|
+
const bytes = part.neighbours.bytesAt(start, end - start);
|
|
67
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
68
|
+
let cursor = 0;
|
|
69
|
+
for (let current = 0; ; current++) {
|
|
70
|
+
if (cursor + 4 > bytes.byteLength) loadFailed(`The neighbour lists of vector ${number} end inside a list`);
|
|
71
|
+
const count = view.getUint32(cursor, true);
|
|
72
|
+
if (cursor + 4 + count * 4 > bytes.byteLength)
|
|
73
|
+
loadFailed(`The neighbour list of vector ${number} runs past its node`);
|
|
74
|
+
if (current === layer) return { view, count, at: cursor + 4 };
|
|
75
|
+
cursor += 4 + count * 4;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
function fetchNeighbourLists(data, numbers) {
|
|
79
|
+
const reads = [];
|
|
80
|
+
for (const number of numbers) {
|
|
81
|
+
const part = data.parts[Math.floor(number / COLD_GRAPH_PART_VECTORS)];
|
|
82
|
+
const offset = number % COLD_GRAPH_PART_VECTORS;
|
|
83
|
+
const start = part.nodeOffsets[offset];
|
|
84
|
+
const size = part.nodeOffsets[offset + 1] - start;
|
|
85
|
+
if (size === 0) continue;
|
|
86
|
+
const read = part.neighbours.fetchRows([start], size);
|
|
87
|
+
if (read !== null) reads.push(read);
|
|
88
|
+
}
|
|
89
|
+
return reads.length === 0 ? null : Promise.all(reads);
|
|
90
|
+
}
|
|
91
|
+
function unseenNeighbours(scope, numbers, layer) {
|
|
92
|
+
const unseen = /* @__PURE__ */ new Set();
|
|
93
|
+
for (const number of numbers) {
|
|
94
|
+
const list = neighbourList(scope.data, number, layer);
|
|
95
|
+
for (let at = 0; at < list.count; at++) {
|
|
96
|
+
const neighbour = list.view.getUint32(list.at + at * 4, true);
|
|
97
|
+
if (neighbour < scope.data.total && scope.visited[neighbour] === 0) unseen.add(neighbour);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return [...unseen];
|
|
101
|
+
}
|
|
102
|
+
function* readTheWalkAhead(scope, next, frontier, layer) {
|
|
103
|
+
const { data } = scope;
|
|
104
|
+
const listOfNext = fetchNeighbourLists(data, [next]);
|
|
105
|
+
const rowsOfNext = listOfNext === null ? scope.fetchDistanceRows(unseenNeighbours(scope, [next], layer)) : null;
|
|
106
|
+
if (listOfNext === null && rowsOfNext === null) return;
|
|
107
|
+
const others = frontier.numbers.slice(0, COLD_WALK_NODES_READ_AHEAD - 1);
|
|
108
|
+
yield Promise.all([listOfNext, rowsOfNext, fetchNeighbourLists(data, others)]);
|
|
109
|
+
const rows = scope.fetchDistanceRows(unseenNeighbours(scope, [next, ...others], layer));
|
|
110
|
+
if (rows !== null) yield rows;
|
|
111
|
+
}
|
|
112
|
+
function* walkLayer(scope, entries, ef, layer, excludes) {
|
|
113
|
+
const { data, distanceOf, isAccepted } = scope;
|
|
114
|
+
const visited = scope.visited;
|
|
115
|
+
visited.fill(0);
|
|
116
|
+
const frontier = { numbers: [], distances: [] };
|
|
117
|
+
const found = { numbers: [], distances: [] };
|
|
118
|
+
function take(number, distance) {
|
|
119
|
+
pushMin(found, number, -distance);
|
|
120
|
+
if (found.numbers.length > ef) popMin(found);
|
|
121
|
+
}
|
|
122
|
+
for (const entry of entries) {
|
|
123
|
+
if (visited[entry] === 1) continue;
|
|
124
|
+
visited[entry] = 1;
|
|
125
|
+
if (data.topLayerOf(entry) < layer) continue;
|
|
126
|
+
const distance = distanceOf(entry);
|
|
127
|
+
if (distance === Number.POSITIVE_INFINITY) continue;
|
|
128
|
+
pushMin(frontier, entry, distance);
|
|
129
|
+
if (excludes && !isAccepted(entry)) continue;
|
|
130
|
+
take(entry, distance);
|
|
131
|
+
}
|
|
132
|
+
let furthest = found.numbers.length > 0 ? negated(found) : Number.POSITIVE_INFINITY;
|
|
133
|
+
for (; ; ) {
|
|
134
|
+
const next = popMin(frontier);
|
|
135
|
+
if (next === null) break;
|
|
136
|
+
if (next.distance > furthest && found.numbers.length >= ef) break;
|
|
137
|
+
if (scope.fetchesRows) yield* readTheWalkAhead(scope, next.number, frontier, layer);
|
|
138
|
+
const list = neighbourList(data, next.number, layer);
|
|
139
|
+
scope.clock?.tick(list.count * data.dimension);
|
|
140
|
+
for (let at = 0; at < list.count; at++) {
|
|
141
|
+
const neighbour = list.view.getUint32(list.at + at * 4, true);
|
|
142
|
+
if (neighbour >= data.total || visited[neighbour] === 1) continue;
|
|
143
|
+
visited[neighbour] = 1;
|
|
144
|
+
if (data.topLayerOf(neighbour) < layer) continue;
|
|
145
|
+
const distance = distanceOf(neighbour);
|
|
146
|
+
if (distance === Number.POSITIVE_INFINITY) continue;
|
|
147
|
+
if (distance < furthest || found.numbers.length < ef) {
|
|
148
|
+
pushMin(frontier, neighbour, distance);
|
|
149
|
+
if (excludes && !isAccepted(neighbour)) continue;
|
|
150
|
+
take(neighbour, distance);
|
|
151
|
+
furthest = negated(found);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return found;
|
|
156
|
+
}
|
|
157
|
+
function nearestFirst(found) {
|
|
158
|
+
const numbers = [];
|
|
159
|
+
const distances = [];
|
|
160
|
+
for (; ; ) {
|
|
161
|
+
const top = popMin(found);
|
|
162
|
+
if (top === null) break;
|
|
163
|
+
numbers.push(top.number);
|
|
164
|
+
distances.push(-top.distance);
|
|
165
|
+
}
|
|
166
|
+
numbers.reverse();
|
|
167
|
+
distances.reverse();
|
|
168
|
+
return { numbers, distances };
|
|
169
|
+
}
|
|
170
|
+
function entryNumberOf(data) {
|
|
171
|
+
const { entryPoint } = data.graph;
|
|
172
|
+
if (entryPoint !== null && data.topLayerOf(entryPoint) >= 0) return entryPoint;
|
|
173
|
+
let best = null;
|
|
174
|
+
let bestTop = -1;
|
|
175
|
+
for (let number = 0; number < data.total; number++) {
|
|
176
|
+
const top = data.topLayerOf(number);
|
|
177
|
+
if (top > bestTop) {
|
|
178
|
+
bestTop = top;
|
|
179
|
+
best = number;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return best;
|
|
183
|
+
}
|
|
184
|
+
function codeDistance(data, query, metric) {
|
|
185
|
+
const { codes, dimension } = data;
|
|
186
|
+
if (codes === null) return null;
|
|
187
|
+
const layout = computeCodeBlockLayout(dimension, codes.bits);
|
|
188
|
+
const code = osqQuantize(query, codes.centroid, layout.queryBits, metric, createOsqScratch(dimension));
|
|
189
|
+
const packed = new Uint8Array(layout.queryBytes);
|
|
190
|
+
stageQueryLevels(code.levels, codes.bits, packed);
|
|
191
|
+
let centroidDot = 0;
|
|
192
|
+
for (let at = 0; at < dimension; at++) centroidDot += codes.centroid[at] * codes.centroid[at];
|
|
193
|
+
const trailer = { lower: 0, upper: 0, correction: 0, sum: 0 };
|
|
194
|
+
const { codeBytes, slotStride, queryBits, bits } = layout;
|
|
195
|
+
const record = new Uint8Array(slotStride);
|
|
196
|
+
const recordView = new DataView(record.buffer);
|
|
197
|
+
return (number) => {
|
|
198
|
+
const file = data.fileOfNumber(number);
|
|
199
|
+
if (file.records === null) return Number.POSITIVE_INFINITY;
|
|
200
|
+
file.records.read((number - file.firstNumber) * slotStride, record);
|
|
201
|
+
const products = scalarQueryProducts(layout, record, 0, packed);
|
|
202
|
+
readTrailer(recordView, 0, codeBytes, trailer);
|
|
203
|
+
const estimate = osqEstimate(products, trailer, bits, code, queryBits, dimension, centroidDot, metric);
|
|
204
|
+
return osqDistance(estimate, metric);
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
function numbersThatCanReachTheTop(scored, k) {
|
|
208
|
+
if (k <= 0 || scored.length <= k) return scored;
|
|
209
|
+
scored.sort((a, b) => b.score - a.score);
|
|
210
|
+
const lowestKept = scored[k - 1].score;
|
|
211
|
+
let end = k;
|
|
212
|
+
while (end < scored.length && scored[end].score === lowestKept) end += 1;
|
|
213
|
+
return scored.slice(0, end);
|
|
214
|
+
}
|
|
215
|
+
function bestHits(hits, k) {
|
|
216
|
+
return hits.sort((a, b) => b.score - a.score || compareCodePoints(a.docId, b.docId)).slice(0, Math.max(k, 0));
|
|
217
|
+
}
|
|
218
|
+
function searchColdGraph(data, query, k, options, accepts, excluded) {
|
|
219
|
+
const walk = walkColdGraph(data, query, k, options, accepts, excluded, false);
|
|
220
|
+
let step = walk.next();
|
|
221
|
+
while (step.done !== true) step = walk.next();
|
|
222
|
+
return step.value;
|
|
223
|
+
}
|
|
224
|
+
async function searchColdGraphConcurrently(data, query, k, options, accepts, excluded) {
|
|
225
|
+
const walk = walkColdGraph(data, query, k, options, accepts, excluded, true);
|
|
226
|
+
let step = walk.next();
|
|
227
|
+
while (step.done !== true) {
|
|
228
|
+
await step.value;
|
|
229
|
+
step = walk.next();
|
|
230
|
+
}
|
|
231
|
+
return step.value;
|
|
232
|
+
}
|
|
233
|
+
function* walkColdGraph(data, query, k, options, accepts, excluded, fetchesRows) {
|
|
234
|
+
const { metric, minSimilarity } = options;
|
|
235
|
+
const entry = entryNumberOf(data);
|
|
236
|
+
if (entry === null) return [];
|
|
237
|
+
const qMag = magnitude(query);
|
|
238
|
+
const fromCodes = codeDistance(data, query, metric);
|
|
239
|
+
const fullDistance = (number) => {
|
|
240
|
+
const vector = data.reusedVectorAt(number);
|
|
241
|
+
return toDistance(query, vector, qMag, metric === "cosine" ? magnitude(vector) : 0, metric);
|
|
242
|
+
};
|
|
243
|
+
const distanceOf = fromCodes ?? fullDistance;
|
|
244
|
+
const acceptance = /* @__PURE__ */ new Map();
|
|
245
|
+
const isAccepted = (number) => {
|
|
246
|
+
if (excluded.has(number) || data.isDeadNumber(number)) return false;
|
|
247
|
+
if (accepts === null) return true;
|
|
248
|
+
const held = acceptance.get(number);
|
|
249
|
+
if (held !== void 0) return held;
|
|
250
|
+
const accepted = accepts(data.idAt(number));
|
|
251
|
+
acceptance.set(number, accepted);
|
|
252
|
+
return accepted;
|
|
253
|
+
};
|
|
254
|
+
const window = options.rescoreWindow ?? k;
|
|
255
|
+
const codes = data.codes;
|
|
256
|
+
let depth = window;
|
|
257
|
+
if (codes !== null) {
|
|
258
|
+
depth = options.oversample === void 0 ? defaultRescoreDepth(codes.bits, data.dimension, window) : Math.ceil(window * options.oversample);
|
|
259
|
+
}
|
|
260
|
+
const ef = Math.max(options.efSearch ?? defaultEfSearch(data.dimension, window), depth);
|
|
261
|
+
const codeRowBytes = codes === null ? null : computeCodeBlockLayout(data.dimension, codes.bits).slotStride;
|
|
262
|
+
const scope = {
|
|
263
|
+
data,
|
|
264
|
+
distanceOf,
|
|
265
|
+
isAccepted,
|
|
266
|
+
visited: new Uint8Array(data.total),
|
|
267
|
+
clock: options.clock,
|
|
268
|
+
fetchesRows,
|
|
269
|
+
fetchDistanceRows: (numbers2) => codeRowBytes === null ? fetchVectorRows(numbers2) : data.fetchRows(numbers2, (file) => file.records, codeRowBytes)
|
|
270
|
+
};
|
|
271
|
+
function fetchVectorRows(numbers2) {
|
|
272
|
+
return data.fetchRows(numbers2, (file) => file.vectors, data.dimension * 4);
|
|
273
|
+
}
|
|
274
|
+
let current = entry;
|
|
275
|
+
for (let layer = Math.min(data.graph.maxLayer, data.topLayerOf(entry)); layer >= 1; layer--) {
|
|
276
|
+
const { numbers: numbers2 } = nearestFirst(yield* walkLayer(scope, [current], 1, layer, false));
|
|
277
|
+
if (numbers2.length > 0) current = numbers2[0];
|
|
278
|
+
}
|
|
279
|
+
const { numbers, distances } = nearestFirst(yield* walkLayer(scope, [current], ef, 0, true));
|
|
280
|
+
const scored = [];
|
|
281
|
+
const rescores = codes !== null;
|
|
282
|
+
const rescoreUntil = rescores ? Math.min(depth, numbers.length) : 0;
|
|
283
|
+
if (fetchesRows && rescoreUntil > 0) {
|
|
284
|
+
const reading = fetchVectorRows(numbers.slice(0, rescoreUntil));
|
|
285
|
+
if (reading !== null) yield reading;
|
|
286
|
+
}
|
|
287
|
+
for (let at = 0; at < numbers.length; at++) {
|
|
288
|
+
const number = numbers[at];
|
|
289
|
+
if (rescores && at >= rescoreUntil && (options.rescoreWindow === void 0 || minSimilarity !== Number.NEGATIVE_INFINITY)) {
|
|
290
|
+
break;
|
|
291
|
+
}
|
|
292
|
+
const distance = at < rescoreUntil ? fullDistance(number) : distances[at];
|
|
293
|
+
const score = toScore(distance, metric);
|
|
294
|
+
if (score < minSimilarity) continue;
|
|
295
|
+
scored.push({ number, score });
|
|
296
|
+
}
|
|
297
|
+
const hits = numbersThatCanReachTheTop(scored, k).map(({ number, score }) => ({ docId: data.idAt(number), score }));
|
|
298
|
+
return bestHits(hits, k);
|
|
299
|
+
}
|
|
300
|
+
var FLOAT64_BYTES = 8;
|
|
301
|
+
var CODES_KEY_AND_MAP_HEADER_BYTES = 7;
|
|
302
|
+
var BIN8_MAX_BYTES = 255;
|
|
303
|
+
var BIN16_MAX_BYTES = 65535;
|
|
304
|
+
var VECTOR_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
305
|
+
var NEIGHBOUR_PAGE_SIZES = /* @__PURE__ */ new Set([RECORD_PAGE_BYTES]);
|
|
306
|
+
var LEVEL_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
|
|
307
|
+
function readHead(path, ref) {
|
|
308
|
+
const { headOffset, headLength, headChecksum } = ref;
|
|
309
|
+
if (headOffset === void 0 || headLength === void 0 || headChecksum === void 0) return null;
|
|
310
|
+
const head = new Uint8Array(headLength);
|
|
311
|
+
readFileRange(path, HEADER_SIZE + headOffset, head);
|
|
312
|
+
if (crc32(head) !== headChecksum) {
|
|
313
|
+
throw new NarsilError(
|
|
314
|
+
ErrorCodes.PERSISTENCE_CRC_MISMATCH,
|
|
315
|
+
`The head of the file "${ref.key}" does not match its checksum`,
|
|
316
|
+
{ key: ref.key }
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
return head;
|
|
320
|
+
}
|
|
321
|
+
function decodedHead(head, key) {
|
|
322
|
+
let raw;
|
|
323
|
+
try {
|
|
324
|
+
raw = decode(head);
|
|
325
|
+
} catch (error) {
|
|
326
|
+
loadFailed(`The head of "${key}" is not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
|
|
327
|
+
}
|
|
328
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) loadFailed(`The head of "${key}" is not a map`);
|
|
329
|
+
return raw;
|
|
330
|
+
}
|
|
331
|
+
function binHeaderBytes(length) {
|
|
332
|
+
if (length <= BIN8_MAX_BYTES) return 2;
|
|
333
|
+
return length <= BIN16_MAX_BYTES ? 3 : 5;
|
|
334
|
+
}
|
|
335
|
+
function isUint32OfThisHost(bytes) {
|
|
336
|
+
return bytes.byteOffset % 4 === 0 && new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
337
|
+
}
|
|
338
|
+
function uint32ValuesOf(bytes) {
|
|
339
|
+
if (isUint32OfThisHost(bytes)) return new Uint32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
340
|
+
const values = new Uint32Array(bytes.byteLength / 4);
|
|
341
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
342
|
+
for (let at = 0; at < values.length; at++) values[at] = view.getUint32(at * 4, true);
|
|
343
|
+
return values;
|
|
344
|
+
}
|
|
345
|
+
function codesOfFile(path, sections, key) {
|
|
346
|
+
const records = sections.get("records");
|
|
347
|
+
const docIds = sections.get("docIds");
|
|
348
|
+
if (records === void 0 || docIds === void 0) return null;
|
|
349
|
+
const start = docIds.offset + docIds.length;
|
|
350
|
+
const length = records.offset - start - binHeaderBytes(records.length);
|
|
351
|
+
if (length <= CODES_KEY_AND_MAP_HEADER_BYTES) loadFailed(`The file "${key}" places its codes before its IDs`);
|
|
352
|
+
const bytes = new Uint8Array(length);
|
|
353
|
+
readFileRange(path, HEADER_SIZE + start, bytes);
|
|
354
|
+
const values = [];
|
|
355
|
+
try {
|
|
356
|
+
for (const value of decodeMulti(bytes.subarray(CODES_KEY_AND_MAP_HEADER_BYTES))) values.push(value);
|
|
357
|
+
} catch (error) {
|
|
358
|
+
loadFailed(`The codes of "${key}" are not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
|
|
359
|
+
}
|
|
360
|
+
const [bitsKey, bits, centroidKey, centroid] = values;
|
|
361
|
+
if (bitsKey !== "bits" || centroidKey !== "centroid" || !Array.isArray(centroid)) {
|
|
362
|
+
loadFailed(`The file "${key}" holds its codes in another layout`);
|
|
363
|
+
}
|
|
364
|
+
if (bits !== 1 && bits !== 2 && bits !== 4 && bits !== 8)
|
|
365
|
+
loadFailed(`The file "${key}" holds ${String(bits)} bit codes`);
|
|
366
|
+
return { bits, centroid: Float32Array.from(centroid.map(Number)) };
|
|
367
|
+
}
|
|
368
|
+
function sameCodes(left, right) {
|
|
369
|
+
if (left.bits !== right.bits || left.centroid.length !== right.centroid.length) return false;
|
|
370
|
+
for (let at = 0; at < left.centroid.length; at++) if (left.centroid[at] !== right.centroid[at]) return false;
|
|
371
|
+
return true;
|
|
372
|
+
}
|
|
373
|
+
function requiredSection(sections, name, key) {
|
|
374
|
+
const section = sections.get(name);
|
|
375
|
+
if (section === void 0) loadFailed(`The head of "${key}" lists no section "${name}"`);
|
|
376
|
+
return section;
|
|
377
|
+
}
|
|
378
|
+
function closeHeldFiles(held) {
|
|
379
|
+
for (const { file } of held) closePagedFile(file);
|
|
380
|
+
}
|
|
381
|
+
function openOneFile(input, at, firstNumber, held, codes) {
|
|
382
|
+
const { ref, path } = input.files[at];
|
|
383
|
+
const head = readHead(path, ref);
|
|
384
|
+
if (head === null) return false;
|
|
385
|
+
const sectionsRaw = decodedHead(head, ref.key).sections;
|
|
386
|
+
const list = decodeSections(sectionsRaw, ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, VECTOR_PAGE_SIZES);
|
|
387
|
+
const sections = new Map(list.map((section) => [section.name, section]));
|
|
388
|
+
const fileCodes = codesOfFile(path, sections, ref.key);
|
|
389
|
+
if (input.codeBits !== null) {
|
|
390
|
+
if (fileCodes === null || fileCodes.bits !== input.codeBits) return false;
|
|
391
|
+
if (codes !== null && !sameCodes(codes, fileCodes)) return false;
|
|
392
|
+
}
|
|
393
|
+
const records = sections.get("records");
|
|
394
|
+
const magnitudeCount = requiredSection(sections, "magnitudes", ref.key).length / FLOAT64_BYTES;
|
|
395
|
+
if (magnitudeCount !== ref.count) loadFailed(`The file "${ref.key}" holds ${magnitudeCount} magnitudes`);
|
|
396
|
+
const ids = requiredSection(sections, "docIds", ref.key);
|
|
397
|
+
const idOffsets = requiredSection(sections, "idOffsets", ref.key);
|
|
398
|
+
const idOrder = requiredSection(sections, "idOrder", ref.key);
|
|
399
|
+
const vectors = requiredSection(sections, "vectors", ref.key);
|
|
400
|
+
const file = openPagedFile(path, readCacheOfThisThread());
|
|
401
|
+
held.push({
|
|
402
|
+
firstNumber,
|
|
403
|
+
count: ref.count,
|
|
404
|
+
dead: ref.dead,
|
|
405
|
+
file,
|
|
406
|
+
ids: createSectionReader(file, ids),
|
|
407
|
+
idOffsets: createSectionReader(file, idOffsets),
|
|
408
|
+
idOrder: createSectionReader(file, idOrder),
|
|
409
|
+
records: records === void 0 ? null : createSectionReader(file, records),
|
|
410
|
+
vectors: createSectionReader(file, vectors)
|
|
411
|
+
});
|
|
412
|
+
return input.codeBits === null ? null : fileCodes;
|
|
413
|
+
}
|
|
414
|
+
function openFiles(input) {
|
|
415
|
+
const files = [];
|
|
416
|
+
let codes = null;
|
|
417
|
+
let firstNumber = 0;
|
|
418
|
+
try {
|
|
419
|
+
for (let at = 0; at < input.files.length; at++) {
|
|
420
|
+
const opened = openOneFile(input, at, firstNumber, files, codes);
|
|
421
|
+
if (opened === false) {
|
|
422
|
+
closeHeldFiles(files);
|
|
423
|
+
return null;
|
|
424
|
+
}
|
|
425
|
+
codes = opened;
|
|
426
|
+
firstNumber += input.files[at].ref.count;
|
|
427
|
+
}
|
|
428
|
+
} catch (error) {
|
|
429
|
+
closeHeldFiles(files);
|
|
430
|
+
throw error;
|
|
431
|
+
}
|
|
432
|
+
return { files, codes };
|
|
433
|
+
}
|
|
434
|
+
function graphLayoutOf(head, ref) {
|
|
435
|
+
const { graphs } = head;
|
|
436
|
+
if (!Array.isArray(graphs) || graphs.length !== 1) return null;
|
|
437
|
+
const layout = graphs[0];
|
|
438
|
+
const payloadLength = ref.bytes - HEADER_SIZE;
|
|
439
|
+
const [levels] = decodeSections([layout.levels], payloadLength, `head of "${ref.key}"`, LEVEL_PAGE_SIZES);
|
|
440
|
+
const [neighbours] = decodeSections([layout.neighbours], payloadLength, `head of "${ref.key}"`, NEIGHBOUR_PAGE_SIZES);
|
|
441
|
+
if (!(layout.nodeOffsets instanceof Uint8Array)) loadFailed(`The head of "${ref.key}" holds no node offsets`);
|
|
442
|
+
const nodeOffsets = uint32ValuesOf(layout.nodeOffsets);
|
|
443
|
+
if (nodeOffsets.length !== levels.length + 1) loadFailed(`The head of "${ref.key}" holds the wrong number of offsets`);
|
|
444
|
+
for (let at = 1; at < nodeOffsets.length; at++) {
|
|
445
|
+
if (nodeOffsets[at] < nodeOffsets[at - 1]) loadFailed(`The head of "${ref.key}" holds descending node offsets`);
|
|
446
|
+
}
|
|
447
|
+
if (nodeOffsets[nodeOffsets.length - 1] > neighbours.length) {
|
|
448
|
+
loadFailed(`The node offsets of "${ref.key}" end outside its neighbour lists`);
|
|
449
|
+
}
|
|
450
|
+
return { entryPoint: layout.entryPoint, maxLayer: layout.maxLayer, m: layout.m, levels, nodeOffsets, neighbours };
|
|
451
|
+
}
|
|
452
|
+
function openParts(input) {
|
|
453
|
+
const parts = [];
|
|
454
|
+
let graph = null;
|
|
455
|
+
try {
|
|
456
|
+
for (const { ref, path } of input.graphParts) {
|
|
457
|
+
const head = readHead(path, ref);
|
|
458
|
+
const layout = head === null ? null : graphLayoutOf(decodedHead(head, ref.key), ref);
|
|
459
|
+
if (layout === null) {
|
|
460
|
+
closeHeldFiles(parts);
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
const { entryPoint, maxLayer, m } = layout;
|
|
464
|
+
if (typeof maxLayer !== "number" || typeof m !== "number") loadFailed(`The head of "${ref.key}" holds no layers`);
|
|
465
|
+
if (entryPoint !== null && typeof entryPoint !== "number") loadFailed(`The head of "${ref.key}" holds no entry`);
|
|
466
|
+
if (graph === null) graph = { entryPoint, maxLayer, m };
|
|
467
|
+
const levels = readWholeSectionChecked(path, layout.levels, false);
|
|
468
|
+
const file = openPagedFile(path, readCacheOfThisThread());
|
|
469
|
+
parts.push({
|
|
470
|
+
file,
|
|
471
|
+
levels,
|
|
472
|
+
nodeOffsets: layout.nodeOffsets,
|
|
473
|
+
neighbours: createSectionReader(file, layout.neighbours)
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
} catch (error) {
|
|
477
|
+
closeHeldFiles(parts);
|
|
478
|
+
throw error;
|
|
479
|
+
}
|
|
480
|
+
return graph === null ? null : { parts, graph };
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// src/persistence/paged/cold-field/index.ts
|
|
484
|
+
var COLD_GRAPH_PART_VECTORS = 16384;
|
|
485
|
+
var FIXSTR = 160;
|
|
486
|
+
var FIXSTR_MAX_BYTES = 31;
|
|
487
|
+
var ASCII_MAX = 127;
|
|
488
|
+
function shortAsciiIdOf(encoded) {
|
|
489
|
+
const length = encoded[0] - FIXSTR;
|
|
490
|
+
if (length < 0 || length > FIXSTR_MAX_BYTES || encoded.length !== length + 1) return null;
|
|
491
|
+
let id = "";
|
|
492
|
+
for (let at = 1; at <= length; at++) {
|
|
493
|
+
const code = encoded[at];
|
|
494
|
+
if (code > ASCII_MAX) return null;
|
|
495
|
+
id += String.fromCharCode(code);
|
|
496
|
+
}
|
|
497
|
+
return id;
|
|
498
|
+
}
|
|
499
|
+
function everyLiveVectorHoldsANode(files, parts) {
|
|
500
|
+
for (const file of files) {
|
|
501
|
+
for (let at = 0; at < file.count; at++) {
|
|
502
|
+
if (isDead(file.dead, at)) continue;
|
|
503
|
+
const number = file.firstNumber + at;
|
|
504
|
+
const part = parts[Math.floor(number / COLD_GRAPH_PART_VECTORS)];
|
|
505
|
+
if (part === void 0) return false;
|
|
506
|
+
if (part.levels[number % COLD_GRAPH_PART_VECTORS] === 0) return false;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return true;
|
|
510
|
+
}
|
|
511
|
+
function createFieldData(input, opened, graph, parts) {
|
|
512
|
+
const { files, codes } = opened;
|
|
513
|
+
const lastFile = files[files.length - 1];
|
|
514
|
+
const total = lastFile === void 0 ? 0 : lastFile.firstNumber + lastFile.count;
|
|
515
|
+
const stride = input.dimension * 4;
|
|
516
|
+
const reusedBytes = new Uint8Array(stride);
|
|
517
|
+
const reusedVector = bytesToVectors(reusedBytes);
|
|
518
|
+
const idOffsetBytes = new Uint8Array(8);
|
|
519
|
+
const idOffsetView = new DataView(idOffsetBytes.buffer);
|
|
520
|
+
function fileOfNumber(number) {
|
|
521
|
+
let low = 0;
|
|
522
|
+
let high = files.length - 1;
|
|
523
|
+
while (low < high) {
|
|
524
|
+
const middle = low + high + 1 >> 1;
|
|
525
|
+
if (files[middle].firstNumber <= number) low = middle;
|
|
526
|
+
else high = middle - 1;
|
|
527
|
+
}
|
|
528
|
+
return files[low];
|
|
529
|
+
}
|
|
530
|
+
function idInFile(file, position) {
|
|
531
|
+
file.idOffsets.read(position * 4, idOffsetBytes);
|
|
532
|
+
const start = idOffsetView.getUint32(0, true);
|
|
533
|
+
const end = idOffsetView.getUint32(4, true);
|
|
534
|
+
if (end < start) loadFailed(`The ID offsets of "${file.file.path}" descend at position ${position}`);
|
|
535
|
+
const encoded = file.ids.bytesAt(start, end - start);
|
|
536
|
+
const id = shortAsciiIdOf(encoded) ?? decode(encoded);
|
|
537
|
+
if (typeof id !== "string") loadFailed(`Position ${position} of "${file.file.path}" holds no ID`);
|
|
538
|
+
return id;
|
|
539
|
+
}
|
|
540
|
+
return {
|
|
541
|
+
dimension: input.dimension,
|
|
542
|
+
metric: input.metric,
|
|
543
|
+
total,
|
|
544
|
+
codes,
|
|
545
|
+
graph,
|
|
546
|
+
files,
|
|
547
|
+
parts,
|
|
548
|
+
fileOfNumber,
|
|
549
|
+
topLayerOf(number) {
|
|
550
|
+
const part = parts[Math.floor(number / COLD_GRAPH_PART_VECTORS)];
|
|
551
|
+
if (part === void 0) return -1;
|
|
552
|
+
const at = number % COLD_GRAPH_PART_VECTORS;
|
|
553
|
+
return at < part.levels.length ? part.levels[at] - 1 : -1;
|
|
554
|
+
},
|
|
555
|
+
idAt(number) {
|
|
556
|
+
const file = fileOfNumber(number);
|
|
557
|
+
return idInFile(file, number - file.firstNumber);
|
|
558
|
+
},
|
|
559
|
+
vectorAt(number) {
|
|
560
|
+
const file = fileOfNumber(number);
|
|
561
|
+
const bytes = new Uint8Array(stride);
|
|
562
|
+
file.vectors.read((number - file.firstNumber) * stride, bytes);
|
|
563
|
+
return bytesToVectors(bytes);
|
|
564
|
+
},
|
|
565
|
+
reusedVectorAt(number) {
|
|
566
|
+
const file = fileOfNumber(number);
|
|
567
|
+
file.vectors.read((number - file.firstNumber) * stride, reusedBytes);
|
|
568
|
+
return reusedVector.buffer === reusedBytes.buffer ? reusedVector : bytesToVectors(reusedBytes);
|
|
569
|
+
},
|
|
570
|
+
isDeadNumber(number) {
|
|
571
|
+
const file = fileOfNumber(number);
|
|
572
|
+
return isDead(file.dead, number - file.firstNumber);
|
|
573
|
+
},
|
|
574
|
+
fetchRows(numbers, sectionOf, rowBytes) {
|
|
575
|
+
const positionsByFile = /* @__PURE__ */ new Map();
|
|
576
|
+
for (const number of numbers) {
|
|
577
|
+
const file = fileOfNumber(number);
|
|
578
|
+
const positions = positionsByFile.get(file);
|
|
579
|
+
const position = (number - file.firstNumber) * rowBytes;
|
|
580
|
+
if (positions === void 0) positionsByFile.set(file, [position]);
|
|
581
|
+
else positions.push(position);
|
|
582
|
+
}
|
|
583
|
+
const reads = [];
|
|
584
|
+
for (const [file, positions] of positionsByFile) {
|
|
585
|
+
const read = sectionOf(file)?.fetchRows(positions, rowBytes) ?? null;
|
|
586
|
+
if (read !== null) reads.push(read);
|
|
587
|
+
}
|
|
588
|
+
return reads.length === 0 ? null : Promise.all(reads);
|
|
589
|
+
}
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
function positionOfId(file, docId, idAt, slot, view) {
|
|
593
|
+
let low = 0;
|
|
594
|
+
let high = file.count - 1;
|
|
595
|
+
while (low <= high) {
|
|
596
|
+
const middle = low + high >> 1;
|
|
597
|
+
file.idOrder.read(middle * 4, slot);
|
|
598
|
+
const position = view.getUint32(0, true);
|
|
599
|
+
if (position >= file.count) loadFailed(`The ID order of "${file.file.path}" holds position ${position}`);
|
|
600
|
+
const order = compareCodePoints(idAt(file.firstNumber + position), docId);
|
|
601
|
+
if (order === 0) return position;
|
|
602
|
+
if (order < 0) low = middle + 1;
|
|
603
|
+
else high = middle - 1;
|
|
604
|
+
}
|
|
605
|
+
return void 0;
|
|
606
|
+
}
|
|
607
|
+
function openColdVectorField(input) {
|
|
608
|
+
const opened = openFiles(input);
|
|
609
|
+
if (opened === null) return null;
|
|
610
|
+
let graphed;
|
|
611
|
+
try {
|
|
612
|
+
graphed = openParts(input);
|
|
613
|
+
} catch (error) {
|
|
614
|
+
closeHeldFiles(opened.files);
|
|
615
|
+
throw error;
|
|
616
|
+
}
|
|
617
|
+
if (graphed === null) {
|
|
618
|
+
closeHeldFiles(opened.files);
|
|
619
|
+
return null;
|
|
620
|
+
}
|
|
621
|
+
const { parts, graph } = graphed;
|
|
622
|
+
function closeAll() {
|
|
623
|
+
closeHeldFiles(opened?.files ?? []);
|
|
624
|
+
closeHeldFiles(parts);
|
|
625
|
+
}
|
|
626
|
+
if (!everyLiveVectorHoldsANode(opened.files, parts)) {
|
|
627
|
+
closeAll();
|
|
628
|
+
return null;
|
|
629
|
+
}
|
|
630
|
+
const data = createFieldData(input, opened, graph, parts);
|
|
631
|
+
let live = 0;
|
|
632
|
+
for (const file of opened.files) {
|
|
633
|
+
for (let at = 0; at < file.count; at++) if (!isDead(file.dead, at)) live += 1;
|
|
634
|
+
}
|
|
635
|
+
let residentBytes = 0;
|
|
636
|
+
for (const part of parts) residentBytes += part.levels.byteLength + part.nodeOffsets.byteLength;
|
|
637
|
+
const orderSlot = new Uint8Array(4);
|
|
638
|
+
const orderView = new DataView(orderSlot.buffer);
|
|
639
|
+
function liveNumberOf(docId) {
|
|
640
|
+
for (let at = data.files.length; at > 0; at--) {
|
|
641
|
+
const file = data.files[at - 1];
|
|
642
|
+
const position = positionOfId(file, docId, data.idAt, orderSlot, orderView);
|
|
643
|
+
if (position !== void 0 && !isDead(file.dead, position)) return file.firstNumber + position;
|
|
644
|
+
}
|
|
645
|
+
return void 0;
|
|
646
|
+
}
|
|
647
|
+
return {
|
|
648
|
+
liveCount: live,
|
|
649
|
+
residentBytes,
|
|
650
|
+
numberOf: liveNumberOf,
|
|
651
|
+
vectorOfNumber: (number) => data.vectorAt(number),
|
|
652
|
+
reusedVectorOfNumber: (number) => data.reusedVectorAt(number),
|
|
653
|
+
partitionOf: (docId) => input.resolvePartition(docId),
|
|
654
|
+
filePaths: () => [...input.files, ...input.graphParts].map((file) => file.path),
|
|
655
|
+
search: (query, k, options, accepts, excluded) => searchColdGraph(data, query, k, options, accepts, excluded),
|
|
656
|
+
searchConcurrently: (query, k, options, accepts, excluded) => searchColdGraphConcurrently(data, query, k, options, accepts, excluded),
|
|
657
|
+
close: closeAll
|
|
658
|
+
};
|
|
659
|
+
}
|
|
660
|
+
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
661
|
+
var FLOAT32_BYTES = 4;
|
|
662
|
+
var FLOAT64_BYTES2 = 8;
|
|
663
|
+
function yieldToSearches() {
|
|
664
|
+
return new Promise((resolve) => setImmediate(resolve));
|
|
665
|
+
}
|
|
666
|
+
function waitBeforeColdLoad(firstSearchAnswered) {
|
|
667
|
+
return new Promise((resolve) => {
|
|
668
|
+
const timer = setTimeout(resolve, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS);
|
|
669
|
+
timer.unref();
|
|
670
|
+
void firstSearchAnswered.then(() => {
|
|
671
|
+
clearTimeout(timer);
|
|
672
|
+
resolve();
|
|
673
|
+
});
|
|
674
|
+
});
|
|
675
|
+
}
|
|
676
|
+
function sectionsOfFile(input) {
|
|
677
|
+
const { ref, path } = input;
|
|
678
|
+
const { headOffset, headLength } = ref;
|
|
679
|
+
if (headOffset === void 0 || headLength === void 0) loadFailed(`The vector file "${ref.key}" holds no head`);
|
|
680
|
+
const head = new Uint8Array(headLength);
|
|
681
|
+
readFileRange(path, HEADER_SIZE + headOffset, head);
|
|
682
|
+
const raw = decode(head);
|
|
683
|
+
const list = decodeSections(raw.sections, ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES);
|
|
684
|
+
return new Map(list.map((section) => [section.name, section]));
|
|
685
|
+
}
|
|
686
|
+
function requiredSection2(sections, name, key) {
|
|
687
|
+
const section = sections.get(name);
|
|
688
|
+
if (section === void 0) loadFailed(`The head of "${key}" lists no section "${name}"`);
|
|
689
|
+
return section;
|
|
690
|
+
}
|
|
691
|
+
function magnitudesOf(bytes) {
|
|
692
|
+
const values = new Float64Array(bytes.byteLength / FLOAT64_BYTES2);
|
|
693
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
694
|
+
for (let at = 0; at < values.length; at++) values[at] = view.getFloat64(at * FLOAT64_BYTES2, true);
|
|
695
|
+
return values;
|
|
696
|
+
}
|
|
697
|
+
async function loadFilesByParts(files, restore, fieldPath, dimension, stillWanted) {
|
|
698
|
+
for (const input of files) {
|
|
699
|
+
await yieldToSearches();
|
|
700
|
+
if (!stillWanted()) return;
|
|
701
|
+
const { ref, path } = input;
|
|
702
|
+
const sections = sectionsOfFile(input);
|
|
703
|
+
const idBytes = readWholeSectionChecked(path, requiredSection2(sections, "docIds", ref.key), false);
|
|
704
|
+
const docIds = decode(idBytes);
|
|
705
|
+
if (!Array.isArray(docIds) || docIds.length !== ref.count) {
|
|
706
|
+
loadFailed(`The vector file "${ref.key}" lists the wrong number of IDs`);
|
|
707
|
+
}
|
|
708
|
+
const recordSection = sections.get("records");
|
|
709
|
+
const records = recordSection === void 0 ? null : readWholeSectionChecked(path, recordSection, false);
|
|
710
|
+
const magnitudes = magnitudesOf(
|
|
711
|
+
readWholeSectionChecked(path, requiredSection2(sections, "magnitudes", ref.key), false)
|
|
712
|
+
);
|
|
713
|
+
const vectorSection = requiredSection2(sections, "vectors", ref.key);
|
|
714
|
+
const stride = dimension * FLOAT32_BYTES;
|
|
715
|
+
await checkEverySectionPage(path, vectorSection, stillWanted);
|
|
716
|
+
if (!stillWanted()) return;
|
|
717
|
+
const fileCodes = codesOfFile(path, sections, ref.key);
|
|
718
|
+
restore.addFile({
|
|
719
|
+
id: ref.id,
|
|
720
|
+
key: ref.key,
|
|
721
|
+
payload: {
|
|
722
|
+
v: VECTOR_FILE_PAYLOAD_VERSION,
|
|
723
|
+
fieldName: fieldPath,
|
|
724
|
+
dimension,
|
|
725
|
+
docIds,
|
|
726
|
+
codes: records === null || fileCodes === null ? null : { bits: fileCodes.bits, centroid: Array.from(fileCodes.centroid), records },
|
|
727
|
+
vectors: new Uint8Array(0)
|
|
728
|
+
},
|
|
729
|
+
dead: ref.dead,
|
|
730
|
+
location: { path, vectorsOffset: HEADER_SIZE + vectorSection.offset },
|
|
731
|
+
byParts: {
|
|
732
|
+
magnitudes,
|
|
733
|
+
vectorAt(position) {
|
|
734
|
+
const bytes = new Uint8Array(stride);
|
|
735
|
+
readFileRange(path, HEADER_SIZE + vectorSection.offset + position * stride, bytes);
|
|
736
|
+
return bytesToVectors(bytes);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
await yieldToSearches();
|
|
742
|
+
if (stillWanted()) restore.finish();
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad };
|