@delali/narsil 0.3.1-next.145 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
- package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
- package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
- package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
- package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
- package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
- package/dist/chunk-3MXV3VHR.mjs +46 -0
- package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
- package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
- package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
- package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
- package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
- package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
- package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/chunk-BTPGQC6Z.mjs +27 -0
- package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
- package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
- package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
- package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
- package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
- package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
- package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
- package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
- package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
- package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
- package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
- package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
- package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +2 -0
- package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
- package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
- package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +150 -255
- package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +539 -5
- package/dist/index.mjs +22 -16
- package/dist/index.react-native.mjs +19 -0
- package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
- package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
- package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
- package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
- package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/react-native/partial-words.mjs +733 -0
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +32 -28
- package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.mjs +7 -5
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
- package/package.json +88 -7
- package/dist/analysis-change-D5APH4Z3.mjs +0 -34
- package/dist/browser/import-FTMLTPGQ.mjs +0 -9
- package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
- package/dist/import-6KXFY6XI.mjs +0 -35
- package/dist/thread-OV6T555H.mjs +0 -28
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-TJMSKBEM.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
3
|
+
|
|
4
|
+
// src/vector/vector-index/dead-bits.ts
|
|
5
|
+
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
6
|
+
function deadBytesFor(count) {
|
|
7
|
+
return Math.ceil(count / 8);
|
|
8
|
+
}
|
|
9
|
+
function isDead(dead, position) {
|
|
10
|
+
return dead !== null && (dead[position >> 3] >> (position & 7) & 1) === 1;
|
|
11
|
+
}
|
|
12
|
+
function countDead(dead, count) {
|
|
13
|
+
if (dead === null) return 0;
|
|
14
|
+
let found = 0;
|
|
15
|
+
for (let position = 0; position < count; position++) {
|
|
16
|
+
if (isDead(dead, position)) found += 1;
|
|
17
|
+
}
|
|
18
|
+
return found;
|
|
19
|
+
}
|
|
20
|
+
function deadBitsWhere(count, isDeadAt) {
|
|
21
|
+
let dead = null;
|
|
22
|
+
for (let position = 0; position < count; position++) {
|
|
23
|
+
if (!isDeadAt(position)) continue;
|
|
24
|
+
dead ??= new Uint8Array(deadBytesFor(count));
|
|
25
|
+
dead[position >> 3] |= 1 << (position & 7);
|
|
26
|
+
}
|
|
27
|
+
return dead;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// src/vector/osq/record.ts
|
|
31
|
+
var OSQ_RECORD_TRAILER_BYTES = 16;
|
|
32
|
+
var TRAILER_LOWER = 0;
|
|
33
|
+
var TRAILER_UPPER = 4;
|
|
34
|
+
var TRAILER_CORRECTION = 8;
|
|
35
|
+
var TRAILER_SUM = 12;
|
|
36
|
+
var QUERY_PLANES_FOR_NARROW_CODES = 4;
|
|
37
|
+
var LEVELS_IN_A_TWO_BIT_BYTE = 4;
|
|
38
|
+
function osqCodeBytes(dimension, bits) {
|
|
39
|
+
return Math.ceil(dimension * bits / 8);
|
|
40
|
+
}
|
|
41
|
+
function osqRecordBytes(dimension, bits) {
|
|
42
|
+
return osqCodeBytes(dimension, bits) + OSQ_RECORD_TRAILER_BYTES;
|
|
43
|
+
}
|
|
44
|
+
function packLevels(levels, bits, target, offset) {
|
|
45
|
+
const dimension = levels.length;
|
|
46
|
+
if (bits === 8) {
|
|
47
|
+
target.set(levels, offset);
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const perByte = 8 / bits;
|
|
51
|
+
target.fill(0, offset, offset + osqCodeBytes(dimension, bits));
|
|
52
|
+
for (let i = 0; i < dimension; i++) {
|
|
53
|
+
const value = levels[i];
|
|
54
|
+
if (value === 0) continue;
|
|
55
|
+
target[offset + Math.floor(i / perByte)] |= value << i % perByte * bits;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function unpackLevels(bytes, offset, dimension, bits) {
|
|
59
|
+
const levels = new Uint8Array(dimension);
|
|
60
|
+
if (bits === 8) {
|
|
61
|
+
levels.set(bytes.subarray(offset, offset + dimension));
|
|
62
|
+
return levels;
|
|
63
|
+
}
|
|
64
|
+
const perByte = 8 / bits;
|
|
65
|
+
const mask = (1 << bits) - 1;
|
|
66
|
+
for (let i = 0; i < dimension; i++) {
|
|
67
|
+
levels[i] = bytes[offset + Math.floor(i / perByte)] >> i % perByte * bits & mask;
|
|
68
|
+
}
|
|
69
|
+
return levels;
|
|
70
|
+
}
|
|
71
|
+
function osqStagedQueryBytes(dimension, documentBits) {
|
|
72
|
+
if (documentBits === 8) return dimension;
|
|
73
|
+
if (documentBits === 4) return osqCodeBytes(dimension, 4);
|
|
74
|
+
return QUERY_PLANES_FOR_NARROW_CODES * osqCodeBytes(dimension, documentBits);
|
|
75
|
+
}
|
|
76
|
+
function stageQueryLevels(levels, documentBits, target) {
|
|
77
|
+
if (documentBits === 8 || documentBits === 4) {
|
|
78
|
+
packLevels(levels, documentBits, target, 0);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const planeBytes = osqCodeBytes(levels.length, documentBits);
|
|
82
|
+
target.fill(0, 0, QUERY_PLANES_FOR_NARROW_CODES * planeBytes);
|
|
83
|
+
if (documentBits === 2) {
|
|
84
|
+
for (let i = 0; i < levels.length; i++) {
|
|
85
|
+
target[i % LEVELS_IN_A_TWO_BIT_BYTE * planeBytes + Math.floor(i / LEVELS_IN_A_TWO_BIT_BYTE)] = levels[i];
|
|
86
|
+
}
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
for (let i = 0; i < levels.length; i++) {
|
|
90
|
+
const value = levels[i];
|
|
91
|
+
if (value === 0) continue;
|
|
92
|
+
const byte = i >> 3;
|
|
93
|
+
const bit = 1 << (i & 7);
|
|
94
|
+
for (let plane = 0; plane < QUERY_PLANES_FOR_NARROW_CODES; plane++) {
|
|
95
|
+
if (value >> plane & 1) target[plane * planeBytes + byte] |= bit;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function writeRecord(target, offset, code, bits, trailer = new DataView(target.buffer, target.byteOffset, target.byteLength)) {
|
|
100
|
+
packLevels(code.levels, bits, target, offset);
|
|
101
|
+
const at = offset + osqCodeBytes(code.levels.length, bits);
|
|
102
|
+
trailer.setFloat32(at + TRAILER_LOWER, code.lower, true);
|
|
103
|
+
trailer.setFloat32(at + TRAILER_UPPER, code.upper, true);
|
|
104
|
+
trailer.setFloat32(at + TRAILER_CORRECTION, code.correction, true);
|
|
105
|
+
trailer.setUint32(at + TRAILER_SUM, code.sum, true);
|
|
106
|
+
}
|
|
107
|
+
function readTrailer(trailer, codeOffset, codeBytes, into) {
|
|
108
|
+
const at = codeOffset + codeBytes;
|
|
109
|
+
into.lower = trailer.getFloat32(at + TRAILER_LOWER, true);
|
|
110
|
+
into.upper = trailer.getFloat32(at + TRAILER_UPPER, true);
|
|
111
|
+
into.correction = trailer.getFloat32(at + TRAILER_CORRECTION, true);
|
|
112
|
+
into.sum = trailer.getUint32(at + TRAILER_SUM, true);
|
|
113
|
+
return into;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// src/vector/vector-index/payload.ts
|
|
117
|
+
var VECTOR_INDEX_PAYLOAD_VERSION = 3;
|
|
118
|
+
var VECTOR_INDEX_PART_VECTORS = 65536;
|
|
119
|
+
var LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
120
|
+
function vectorsToBytes(vectors) {
|
|
121
|
+
if (LITTLE_ENDIAN) return new Uint8Array(vectors.buffer, vectors.byteOffset, vectors.byteLength);
|
|
122
|
+
const bytes = new Uint8Array(vectors.byteLength);
|
|
123
|
+
const view = new DataView(bytes.buffer);
|
|
124
|
+
for (let i = 0; i < vectors.length; i++) view.setFloat32(i * 4, vectors[i], true);
|
|
125
|
+
return bytes;
|
|
126
|
+
}
|
|
127
|
+
function bytesToVectors(bytes) {
|
|
128
|
+
if (LITTLE_ENDIAN && bytes.byteOffset % 4 === 0) {
|
|
129
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
130
|
+
}
|
|
131
|
+
const vectors = new Float32Array(bytes.byteLength / 4);
|
|
132
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
133
|
+
for (let i = 0; i < vectors.length; i++) vectors[i] = view.getFloat32(i * 4, true);
|
|
134
|
+
return vectors;
|
|
135
|
+
}
|
|
136
|
+
function invalid(message, details) {
|
|
137
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid vector index payload: ${message}`, details);
|
|
138
|
+
}
|
|
139
|
+
function isRecord(value) {
|
|
140
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
141
|
+
}
|
|
142
|
+
function isNonNegativeInteger(value) {
|
|
143
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
144
|
+
}
|
|
145
|
+
function isSerializedGraph(value) {
|
|
146
|
+
return isRecord(value) && Array.isArray(value.nodes);
|
|
147
|
+
}
|
|
148
|
+
function codesOf(raw, dimension, count) {
|
|
149
|
+
if (raw === null || raw === void 0) return null;
|
|
150
|
+
if (!isRecord(raw)) invalid("codes must be a map or nil");
|
|
151
|
+
const bits = raw.bits;
|
|
152
|
+
if (bits !== 1 && bits !== 2 && bits !== 4 && bits !== 8) invalid("codes.bits must be 8, 4, 2, or 1", { bits });
|
|
153
|
+
if (!Array.isArray(raw.centroid) || raw.centroid.length !== dimension) {
|
|
154
|
+
invalid("codes.centroid must hold one number per dimension");
|
|
155
|
+
}
|
|
156
|
+
if (!(raw.records instanceof Uint8Array)) invalid("codes.records must be bytes");
|
|
157
|
+
if (raw.records.byteLength !== count * osqRecordBytes(dimension, bits)) {
|
|
158
|
+
invalid("codes.records must hold one record per vector", { count, bytes: raw.records.byteLength });
|
|
159
|
+
}
|
|
160
|
+
return { bits, centroid: raw.centroid.map(Number), records: raw.records };
|
|
161
|
+
}
|
|
162
|
+
function decodeVectorIndexPart(raw) {
|
|
163
|
+
if (!isRecord(raw)) invalid("payload must be a map");
|
|
164
|
+
if (raw.v !== VECTOR_INDEX_PAYLOAD_VERSION) {
|
|
165
|
+
throw new NarsilError(
|
|
166
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
167
|
+
`This vector index payload was written in layout ${String(raw.v ?? 1)}, and this build reads layout ${VECTOR_INDEX_PAYLOAD_VERSION} alone`,
|
|
168
|
+
{ version: raw.v ?? 1 }
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
if (typeof raw.fieldName !== "string") invalid("fieldName must be a string");
|
|
172
|
+
if (!isNonNegativeInteger(raw.dimension) || raw.dimension === 0) invalid("dimension must be a positive integer");
|
|
173
|
+
if (!isNonNegativeInteger(raw.part)) invalid("part must be a non-negative integer");
|
|
174
|
+
if (!isNonNegativeInteger(raw.parts) || raw.parts === 0 || raw.part >= raw.parts) {
|
|
175
|
+
invalid("parts must exceed part", { part: raw.part, parts: raw.parts });
|
|
176
|
+
}
|
|
177
|
+
if (!Array.isArray(raw.docIds) || !raw.docIds.every((docId) => typeof docId === "string")) {
|
|
178
|
+
invalid("docIds must be a list of strings");
|
|
179
|
+
}
|
|
180
|
+
if (raw.docIds.length > VECTOR_INDEX_PART_VECTORS) {
|
|
181
|
+
invalid(`a part holds at most ${VECTOR_INDEX_PART_VECTORS} vectors`, { count: raw.docIds.length });
|
|
182
|
+
}
|
|
183
|
+
if (!Array.isArray(raw.graphs) || !raw.graphs.every(isSerializedGraph)) {
|
|
184
|
+
invalid("graphs must be a list of graphs, each holding a list of nodes");
|
|
185
|
+
}
|
|
186
|
+
if (!(raw.vectors instanceof Uint8Array)) invalid("vectors must be bytes");
|
|
187
|
+
if (raw.vectors.byteLength !== raw.docIds.length * raw.dimension * 4) {
|
|
188
|
+
invalid("vectors must hold dimension float32 components per document", {
|
|
189
|
+
count: raw.docIds.length,
|
|
190
|
+
bytes: raw.vectors.byteLength
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return {
|
|
194
|
+
v: VECTOR_INDEX_PAYLOAD_VERSION,
|
|
195
|
+
fieldName: raw.fieldName,
|
|
196
|
+
dimension: raw.dimension,
|
|
197
|
+
part: raw.part,
|
|
198
|
+
parts: raw.parts,
|
|
199
|
+
docIds: raw.docIds,
|
|
200
|
+
graphs: raw.graphs,
|
|
201
|
+
codes: codesOf(raw.codes, raw.dimension, raw.docIds.length),
|
|
202
|
+
vectors: raw.vectors
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
function decodeVectorIndexParts(raw) {
|
|
206
|
+
if (!Array.isArray(raw)) invalid("a field must carry a list of parts");
|
|
207
|
+
return raw.map(decodeVectorIndexPart);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// src/vector/vector-index/checkpoint-payload.ts
|
|
211
|
+
var VECTOR_FILE_PAYLOAD_VERSION = 1;
|
|
212
|
+
var VECTOR_GRAPH_PAYLOAD_VERSION = 1;
|
|
213
|
+
var VECTOR_GRAPH_PART_PAYLOAD_VERSION = 2;
|
|
214
|
+
var VECTOR_GRAPH_PART_VECTORS = 16384;
|
|
215
|
+
var VECTOR_FILE_MAX_VECTORS = VECTOR_INDEX_PART_VECTORS;
|
|
216
|
+
var METRICS = ["cosine", "dotProduct", "euclidean"];
|
|
217
|
+
function invalid2(payload, message, details) {
|
|
218
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid ${payload} payload: ${message}`, details);
|
|
219
|
+
}
|
|
220
|
+
function isRecord2(value) {
|
|
221
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
222
|
+
}
|
|
223
|
+
function isIntegerUpTo(value, highest) {
|
|
224
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= highest;
|
|
225
|
+
}
|
|
226
|
+
function requireVersion(payload, found, expected) {
|
|
227
|
+
if (found === expected) return;
|
|
228
|
+
throw new NarsilError(
|
|
229
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
230
|
+
`This ${payload} payload was written in layout ${String(found)}, and this build reads layout ${expected} alone`,
|
|
231
|
+
{ version: found }
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
function decodeVectorFilePayload(raw) {
|
|
235
|
+
const name = "vector file";
|
|
236
|
+
if (!isRecord2(raw)) invalid2(name, "payload must be a map");
|
|
237
|
+
requireVersion(name, raw.v, VECTOR_FILE_PAYLOAD_VERSION);
|
|
238
|
+
if (typeof raw.fieldName !== "string") invalid2(name, "fieldName must be a string");
|
|
239
|
+
if (!isIntegerUpTo(raw.dimension, 65535) || raw.dimension === 0) invalid2(name, "dimension must be a positive uint16");
|
|
240
|
+
if (!Array.isArray(raw.docIds) || !raw.docIds.every((docId) => typeof docId === "string")) {
|
|
241
|
+
invalid2(name, "docIds must be a list of strings");
|
|
242
|
+
}
|
|
243
|
+
if (raw.docIds.length > VECTOR_FILE_MAX_VECTORS) {
|
|
244
|
+
invalid2(name, `a file holds at most ${VECTOR_FILE_MAX_VECTORS} vectors`, { count: raw.docIds.length });
|
|
245
|
+
}
|
|
246
|
+
if (!(raw.vectors instanceof Uint8Array)) invalid2(name, "vectors must be bytes");
|
|
247
|
+
if (raw.vectors.byteLength !== raw.docIds.length * raw.dimension * 4) {
|
|
248
|
+
invalid2(name, "vectors must hold dimension float32 components per document", {
|
|
249
|
+
count: raw.docIds.length,
|
|
250
|
+
bytes: raw.vectors.byteLength
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
return {
|
|
254
|
+
v: VECTOR_FILE_PAYLOAD_VERSION,
|
|
255
|
+
fieldName: raw.fieldName,
|
|
256
|
+
dimension: raw.dimension,
|
|
257
|
+
docIds: raw.docIds,
|
|
258
|
+
codes: codesOf(raw.codes, raw.dimension, raw.docIds.length),
|
|
259
|
+
vectors: raw.vectors
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
function isMetric(value) {
|
|
263
|
+
return METRICS.some((metric) => metric === value);
|
|
264
|
+
}
|
|
265
|
+
function calibratedMeanLengthOf(name, value) {
|
|
266
|
+
if (value === void 0 || value === null) return null;
|
|
267
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
268
|
+
invalid2(name, "calibratedMeanLength must be a finite float64 of at least 0");
|
|
269
|
+
}
|
|
270
|
+
return value;
|
|
271
|
+
}
|
|
272
|
+
function decodeNumberedGraph(raw, minimumM) {
|
|
273
|
+
const name = "vector graph";
|
|
274
|
+
if (!isRecord2(raw)) invalid2(name, "each graph must be a map");
|
|
275
|
+
const entryPoint = raw.entryPoint ?? null;
|
|
276
|
+
if (entryPoint !== null && !isIntegerUpTo(entryPoint, 4294967295))
|
|
277
|
+
invalid2(name, "entryPoint must be a uint32 or nil");
|
|
278
|
+
if (!isIntegerUpTo(raw.maxLayer, 255)) invalid2(name, "maxLayer must be a uint8");
|
|
279
|
+
if (!isIntegerUpTo(raw.m, MAX_M) || raw.m < minimumM) {
|
|
280
|
+
invalid2(name, `m must be a whole number from ${minimumM} to ${MAX_M}`);
|
|
281
|
+
}
|
|
282
|
+
if (!isIntegerUpTo(raw.efConstruction, 65535)) invalid2(name, "efConstruction must be a uint16");
|
|
283
|
+
if (!isMetric(raw.metric)) invalid2(name, "metric must name a supported metric", { metric: raw.metric });
|
|
284
|
+
if (!(raw.levels instanceof Uint8Array)) invalid2(name, "levels must be bytes");
|
|
285
|
+
if (!(raw.neighbours instanceof Uint8Array)) invalid2(name, "neighbours must be bytes");
|
|
286
|
+
const graph = {
|
|
287
|
+
entryPoint,
|
|
288
|
+
maxLayer: raw.maxLayer,
|
|
289
|
+
m: raw.m,
|
|
290
|
+
efConstruction: raw.efConstruction,
|
|
291
|
+
metric: raw.metric,
|
|
292
|
+
levels: raw.levels,
|
|
293
|
+
neighbours: raw.neighbours
|
|
294
|
+
};
|
|
295
|
+
const calibratedMeanLength = calibratedMeanLengthOf(name, raw.calibratedMeanLength);
|
|
296
|
+
const shuffledBuildCount = raw.shuffledBuildCount ?? null;
|
|
297
|
+
if (shuffledBuildCount === null) return graph;
|
|
298
|
+
if (!isIntegerUpTo(shuffledBuildCount, 4294967295)) invalid2(name, "shuffledBuildCount must be a uint32");
|
|
299
|
+
if (calibratedMeanLength === null) return { ...graph, shuffledBuildCount };
|
|
300
|
+
return { ...graph, shuffledBuildCount, calibratedMeanLength };
|
|
301
|
+
}
|
|
302
|
+
function decodeVectorGraphPayload(raw) {
|
|
303
|
+
const name = "vector graph";
|
|
304
|
+
if (!isRecord2(raw)) invalid2(name, "payload must be a map");
|
|
305
|
+
requireVersion(name, raw.v, VECTOR_GRAPH_PAYLOAD_VERSION);
|
|
306
|
+
if (typeof raw.fieldName !== "string") invalid2(name, "fieldName must be a string");
|
|
307
|
+
if (!Array.isArray(raw.graphs)) invalid2(name, "graphs must be a list");
|
|
308
|
+
return {
|
|
309
|
+
v: VECTOR_GRAPH_PAYLOAD_VERSION,
|
|
310
|
+
fieldName: raw.fieldName,
|
|
311
|
+
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph, EARLIER_RELEASE_MIN_M))
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
function decodeVectorGraphPartPayload(raw) {
|
|
315
|
+
const name = "vector graph part";
|
|
316
|
+
if (!isRecord2(raw)) invalid2(name, "payload must be a map");
|
|
317
|
+
requireVersion(name, raw.v, VECTOR_GRAPH_PART_PAYLOAD_VERSION);
|
|
318
|
+
if (typeof raw.fieldName !== "string") invalid2(name, "fieldName must be a string");
|
|
319
|
+
if (!Array.isArray(raw.graphs)) invalid2(name, "graphs must be a list");
|
|
320
|
+
return {
|
|
321
|
+
v: VECTOR_GRAPH_PART_PAYLOAD_VERSION,
|
|
322
|
+
fieldName: raw.fieldName,
|
|
323
|
+
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph, MIN_M))
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
function sameGraphHeader(a, b) {
|
|
327
|
+
return a.entryPoint === b.entryPoint && a.maxLayer === b.maxLayer && a.m === b.m && a.efConstruction === b.efConstruction && a.metric === b.metric && a.shuffledBuildCount === b.shuffledBuildCount && a.calibratedMeanLength === b.calibratedMeanLength;
|
|
328
|
+
}
|
|
329
|
+
function assembleGraphParts(parts) {
|
|
330
|
+
const name = "vector graph part";
|
|
331
|
+
const first = parts[0];
|
|
332
|
+
if (first === void 0) invalid2(name, "a field that holds a graph must have at least one part");
|
|
333
|
+
const partLists = first.graphs.map(() => []);
|
|
334
|
+
for (let p = 0; p < parts.length; p++) {
|
|
335
|
+
const part = parts[p];
|
|
336
|
+
const isLast = p === parts.length - 1;
|
|
337
|
+
if (part.fieldName !== first.fieldName || part.graphs.length !== first.graphs.length) {
|
|
338
|
+
invalid2(name, `part ${p} lists other graphs than part 0`, { part: p });
|
|
339
|
+
}
|
|
340
|
+
for (let g = 0; g < part.graphs.length; g++) {
|
|
341
|
+
const graph = part.graphs[g];
|
|
342
|
+
if (!sameGraphHeader(graph, first.graphs[g]))
|
|
343
|
+
invalid2(name, `part ${p} gives graph ${g} other headers`, { part: p });
|
|
344
|
+
const levels = graph.levels.length;
|
|
345
|
+
const fits = isLast ? levels >= 1 && levels <= VECTOR_GRAPH_PART_VECTORS : levels === VECTOR_GRAPH_PART_VECTORS;
|
|
346
|
+
if (!fits) invalid2(name, `part ${p} gives graph ${g} ${levels} bytes of levels`, { part: p, levels });
|
|
347
|
+
partLists[g].push(graph);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
const graphs = partLists.map((list) => ({ partSize: VECTOR_GRAPH_PART_VECTORS, parts: list }));
|
|
351
|
+
return { fieldName: first.fieldName, graphs };
|
|
352
|
+
}
|
|
353
|
+
function graphPartsOfEarlierPayload(payload) {
|
|
354
|
+
return {
|
|
355
|
+
fieldName: payload.fieldName,
|
|
356
|
+
graphs: payload.graphs.map((graph) => ({ partSize: Math.max(graph.levels.length, 1), parts: [graph] }))
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
export { NO_LIVE_VECTOR_AT_POSITION, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_GRAPH_PART_VECTORS, assembleGraphParts, bytesToVectors, countDead, deadBitsWhere, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, graphPartsOfEarlierPayload, isDead, osqCodeBytes, osqRecordBytes, osqStagedQueryBytes, readTrailer, stageQueryLevels, unpackLevels, vectorsToBytes, writeRecord };
|