@delali/narsil 0.3.1-next.144 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
- package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/chunk-3TVDTACC.mjs +104 -0
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
- package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/chunk-6BWLDLS3.mjs +58 -0
- package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
- package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/chunk-B7XIOAMK.mjs +100 -0
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
- package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
- package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
- package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
- package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
- package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
- package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/chunk-SOUZMPME.mjs +1635 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
- package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
- package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
- package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
- package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/chunk-YKEHQUSZ.mjs +248 -0
- package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +46 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
- package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16116 -14537
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +542 -8
- package/dist/index.mjs +29 -32
- package/dist/index.react-native.mjs +19 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
- package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
- package/dist/module-V5K4ZP23.mjs +45 -0
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +11 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +159 -38
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
- package/openapi.json +349 -2
- package/package.json +108 -8
- package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
- package/dist/module-IYPLWHGC.mjs +0 -48
- package/dist/thread-KGNAIKPD.mjs +0 -33
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-YJSMEKRE.mjs';
|
|
2
|
+
import { getNestedValue, requireValidFieldTypes } from './chunk-PU55IOG4.mjs';
|
|
3
|
+
import { VECTOR_PATTERN } from './chunk-DKJTZ625.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
|
|
5
|
+
import { decode } from '@msgpack/msgpack';
|
|
6
|
+
|
|
7
|
+
function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes, schedulesGraphBuild = true) {
|
|
8
|
+
if (entry.document === null) {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
const document = decode(entry.document);
|
|
12
|
+
restoreVectorFields(document, vectorFieldPaths);
|
|
13
|
+
const documentId = entry.documentId;
|
|
14
|
+
if (manager.has(documentId)) {
|
|
15
|
+
removeDocumentVectors(documentId, vecIndexes);
|
|
16
|
+
manager.remove(documentId);
|
|
17
|
+
}
|
|
18
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths);
|
|
19
|
+
manager.insert(documentId, partitionDoc);
|
|
20
|
+
try {
|
|
21
|
+
insertDocumentVectors(documentId, extractedVectors, vecIndexes, manager.partitionIdOf(documentId));
|
|
22
|
+
} catch (err) {
|
|
23
|
+
manager.remove(documentId);
|
|
24
|
+
throw err;
|
|
25
|
+
}
|
|
26
|
+
if (!schedulesGraphBuild) return;
|
|
27
|
+
for (const fieldPath of extractedVectors.keys()) {
|
|
28
|
+
const vecIndex = vecIndexes.get(fieldPath);
|
|
29
|
+
if (vecIndex) {
|
|
30
|
+
vecIndex.scheduleBuild();
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
|
|
35
|
+
removeDocumentVectors(entry.documentId, vecIndexes);
|
|
36
|
+
if (!manager.has(entry.documentId)) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
manager.remove(entry.documentId, removedSpellings);
|
|
40
|
+
}
|
|
41
|
+
function restoreVectorFields(document, vectorFieldPaths) {
|
|
42
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
43
|
+
const value = getNestedValue(document, fieldPath);
|
|
44
|
+
if (value instanceof Uint8Array && !(value instanceof Float32Array)) {
|
|
45
|
+
if (value.byteLength % 4 !== 0) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
`Vector field "${fieldPath}" has invalid byte length ${value.byteLength} (must be divisible by 4)`
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const elementCount = value.byteLength / 4;
|
|
51
|
+
const float32 = new Float32Array(elementCount);
|
|
52
|
+
const dataView = new DataView(value.buffer, value.byteOffset, value.byteLength);
|
|
53
|
+
for (let i = 0; i < elementCount; i += 1) {
|
|
54
|
+
float32[i] = dataView.getFloat32(i * 4, true);
|
|
55
|
+
}
|
|
56
|
+
setNestedValue(document, fieldPath, float32);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
61
|
+
function setNestedValue(obj, path, value) {
|
|
62
|
+
if (!path.includes(".")) {
|
|
63
|
+
if (FORBIDDEN_KEYS.has(path)) {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
obj[path] = value;
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const segments = path.split(".");
|
|
70
|
+
let current = obj;
|
|
71
|
+
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
72
|
+
const segment = segments[i];
|
|
73
|
+
if (segment === void 0 || FORBIDDEN_KEYS.has(segment)) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
let next = current[segment];
|
|
77
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
78
|
+
next = {};
|
|
79
|
+
current[segment] = next;
|
|
80
|
+
}
|
|
81
|
+
current = next;
|
|
82
|
+
}
|
|
83
|
+
const finalSegment = segments.at(-1);
|
|
84
|
+
if (finalSegment === void 0 || FORBIDDEN_KEYS.has(finalSegment)) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
current[finalSegment] = value;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// src/serialization/keyword-segment/types.ts
|
|
91
|
+
var KEYWORD_SEGMENT_PAYLOAD_VERSION = 1;
|
|
92
|
+
var KEYWORD_SEGMENT_ENVELOPE_VERSION = 4;
|
|
93
|
+
var MAX_SEGMENT_FIELD_NAMES = 65536;
|
|
94
|
+
|
|
95
|
+
// src/serialization/keyword-segment/little-endian.ts
|
|
96
|
+
var LITTLE_ENDIAN_PLATFORM = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
97
|
+
function swappedCopy(values) {
|
|
98
|
+
const width = values.BYTES_PER_ELEMENT;
|
|
99
|
+
const source = new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
100
|
+
const copy = new Uint8Array(values.byteLength);
|
|
101
|
+
for (let at = 0; at < copy.length; at += width) {
|
|
102
|
+
for (let k = 0; k < width; k++) copy[at + k] = source[at + width - 1 - k];
|
|
103
|
+
}
|
|
104
|
+
return copy;
|
|
105
|
+
}
|
|
106
|
+
function littleEndianBytes(values) {
|
|
107
|
+
if (!LITTLE_ENDIAN_PLATFORM) return swappedCopy(values);
|
|
108
|
+
return new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
109
|
+
}
|
|
110
|
+
function allocate(byteLength, shared) {
|
|
111
|
+
return shared ? new SharedArrayBuffer(byteLength) : new ArrayBuffer(byteLength);
|
|
112
|
+
}
|
|
113
|
+
function copyInto(bytes, width, shared) {
|
|
114
|
+
const buffer = allocate(bytes.byteLength, shared);
|
|
115
|
+
const target = new Uint8Array(buffer);
|
|
116
|
+
if (LITTLE_ENDIAN_PLATFORM) {
|
|
117
|
+
target.set(bytes);
|
|
118
|
+
return buffer;
|
|
119
|
+
}
|
|
120
|
+
for (let at = 0; at < bytes.length; at += width) {
|
|
121
|
+
for (let k = 0; k < width; k++) target[at + k] = bytes[at + width - 1 - k];
|
|
122
|
+
}
|
|
123
|
+
return buffer;
|
|
124
|
+
}
|
|
125
|
+
function uint32sFrom(bytes, shared) {
|
|
126
|
+
return new Uint32Array(copyInto(bytes, 4, shared));
|
|
127
|
+
}
|
|
128
|
+
function uint16sFrom(bytes, shared) {
|
|
129
|
+
return new Uint16Array(copyInto(bytes, 2, shared));
|
|
130
|
+
}
|
|
131
|
+
function float64sFrom(bytes, shared) {
|
|
132
|
+
return new Float64Array(copyInto(bytes, 8, shared));
|
|
133
|
+
}
|
|
134
|
+
function bytesFrom(bytes, shared) {
|
|
135
|
+
return new Uint8Array(copyInto(bytes, 1, shared));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// src/serialization/keyword-segment/raw-fields.ts
|
|
139
|
+
function corrupt(message, details) {
|
|
140
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid keyword segment: ${message}`, details);
|
|
141
|
+
}
|
|
142
|
+
function isRecord(value) {
|
|
143
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && !(value instanceof Uint8Array);
|
|
144
|
+
}
|
|
145
|
+
function requireRecord(value, name) {
|
|
146
|
+
if (!isRecord(value)) corrupt(`${name} must be a map`);
|
|
147
|
+
return value;
|
|
148
|
+
}
|
|
149
|
+
function requireList(value, name) {
|
|
150
|
+
if (!Array.isArray(value)) corrupt(`${name} must be a list`);
|
|
151
|
+
return value;
|
|
152
|
+
}
|
|
153
|
+
function requireString(value, name) {
|
|
154
|
+
if (typeof value !== "string") corrupt(`${name} must be a string`);
|
|
155
|
+
return value;
|
|
156
|
+
}
|
|
157
|
+
function requireStrings(value, name) {
|
|
158
|
+
const list = requireList(value, name);
|
|
159
|
+
for (const item of list) {
|
|
160
|
+
if (typeof item !== "string") corrupt(`${name} must hold strings alone`);
|
|
161
|
+
}
|
|
162
|
+
return list;
|
|
163
|
+
}
|
|
164
|
+
function requireBytes(value, name, width) {
|
|
165
|
+
if (!(value instanceof Uint8Array)) corrupt(`${name} must be bytes`);
|
|
166
|
+
if (value.byteLength % width !== 0) {
|
|
167
|
+
corrupt(`${name} holds ${value.byteLength} bytes, which is not a whole number of ${width}-byte values`, {
|
|
168
|
+
field: name
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
return value;
|
|
172
|
+
}
|
|
173
|
+
function requireBytesOrNil(value, name, width) {
|
|
174
|
+
if (value === null || value === void 0) return null;
|
|
175
|
+
return requireBytes(value, name, width);
|
|
176
|
+
}
|
|
177
|
+
function requireUint32(value, name) {
|
|
178
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > 4294967295) {
|
|
179
|
+
corrupt(`${name} must be a uint32`);
|
|
180
|
+
}
|
|
181
|
+
return value;
|
|
182
|
+
}
|
|
183
|
+
function requireCount(found, expected, name) {
|
|
184
|
+
if (found === expected) return;
|
|
185
|
+
corrupt(`${name} holds ${found} values where the segment needs ${expected}`, { field: name, found, expected });
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// src/serialization/keyword-segment/utf8.ts
|
|
189
|
+
var CONTINUATION_MASK = 192;
|
|
190
|
+
var CONTINUATION_BYTE = 128;
|
|
191
|
+
function continues(bytes, at, count) {
|
|
192
|
+
if (at + count > bytes.length) return false;
|
|
193
|
+
for (let k = 1; k <= count; k++) {
|
|
194
|
+
if ((bytes[at + k] & CONTINUATION_MASK) !== CONTINUATION_BYTE) return false;
|
|
195
|
+
}
|
|
196
|
+
return true;
|
|
197
|
+
}
|
|
198
|
+
function sequenceLengthAt(bytes, at) {
|
|
199
|
+
const lead = bytes[at];
|
|
200
|
+
if (lead < 128) return 1;
|
|
201
|
+
if (lead >= 194 && lead <= 223) return continues(bytes, at, 1) ? 2 : 0;
|
|
202
|
+
if (lead >= 224 && lead <= 239) {
|
|
203
|
+
if (!continues(bytes, at, 2)) return 0;
|
|
204
|
+
const second = bytes[at + 1];
|
|
205
|
+
if (lead === 224 && second < 160) return 0;
|
|
206
|
+
if (lead === 237 && second > 159) return 0;
|
|
207
|
+
return 3;
|
|
208
|
+
}
|
|
209
|
+
if (lead >= 240 && lead <= 244) {
|
|
210
|
+
if (!continues(bytes, at, 3)) return 0;
|
|
211
|
+
const second = bytes[at + 1];
|
|
212
|
+
if (lead === 240 && second < 144) return 0;
|
|
213
|
+
if (lead === 244 && second > 143) return 0;
|
|
214
|
+
return 4;
|
|
215
|
+
}
|
|
216
|
+
return 0;
|
|
217
|
+
}
|
|
218
|
+
function isValidUtf8(bytes) {
|
|
219
|
+
let at = 0;
|
|
220
|
+
while (at < bytes.length) {
|
|
221
|
+
const length = sequenceLengthAt(bytes, at);
|
|
222
|
+
if (length === 0) return false;
|
|
223
|
+
at += length;
|
|
224
|
+
}
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// src/serialization/keyword-segment/validate.ts
|
|
229
|
+
var CONTINUATION_MASK2 = 192;
|
|
230
|
+
var CONTINUATION_BYTE2 = 128;
|
|
231
|
+
function requireOffsets(offsets, listSize, name) {
|
|
232
|
+
if (offsets.length === 0) corrupt(`${name} must hold at least one value`, { field: name });
|
|
233
|
+
if (offsets[0] !== 0) corrupt(`${name} must start at 0`, { field: name });
|
|
234
|
+
for (let at = 1; at < offsets.length; at++) {
|
|
235
|
+
if (offsets[at] < offsets[at - 1]) corrupt(`${name} decreases at entry ${at}`, { field: name, at });
|
|
236
|
+
}
|
|
237
|
+
if (offsets[offsets.length - 1] !== listSize) {
|
|
238
|
+
corrupt(`${name} ends at ${offsets[offsets.length - 1]} where the list it divides holds ${listSize}`, {
|
|
239
|
+
field: name
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
function requireUtf8Entries(blob, offsets, name) {
|
|
244
|
+
if (!isValidUtf8(blob)) corrupt(`${name} is not valid UTF-8`, { field: name });
|
|
245
|
+
for (let at = 0; at < offsets.length; at++) {
|
|
246
|
+
const offset = offsets[at];
|
|
247
|
+
if (offset < blob.length && (blob[offset] & CONTINUATION_MASK2) === CONTINUATION_BYTE2) {
|
|
248
|
+
corrupt(`${name} splits a character at entry ${at}`, { field: name, at });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function requireOrdinals(values, documentCount, name) {
|
|
253
|
+
for (let at = 0; at < values.length; at++) {
|
|
254
|
+
if (values[at] >= documentCount) {
|
|
255
|
+
corrupt(`${name} names document ${values[at]} of a segment of ${documentCount}`, { field: name, at });
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
function requireStrictlyAscending(values, start, end, name) {
|
|
260
|
+
for (let at = start + 1; at < end; at++) {
|
|
261
|
+
if (values[at] <= values[at - 1]) corrupt(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
function compareEntries(blob, offsets, left, right) {
|
|
265
|
+
const leftStart = offsets[left];
|
|
266
|
+
const leftLength = offsets[left + 1] - leftStart;
|
|
267
|
+
const rightStart = offsets[right];
|
|
268
|
+
const rightLength = offsets[right + 1] - rightStart;
|
|
269
|
+
const shared = leftLength < rightLength ? leftLength : rightLength;
|
|
270
|
+
for (let k = 0; k < shared; k++) {
|
|
271
|
+
const diff = blob[leftStart + k] - blob[rightStart + k];
|
|
272
|
+
if (diff !== 0) return diff;
|
|
273
|
+
}
|
|
274
|
+
return leftLength - rightLength;
|
|
275
|
+
}
|
|
276
|
+
function requireEntriesAscend(blob, offsets, order, name) {
|
|
277
|
+
const count = offsets.length - 1;
|
|
278
|
+
for (let at = 1; at < count; at++) {
|
|
279
|
+
const left = order === null ? at - 1 : order[at - 1];
|
|
280
|
+
const right = order === null ? at : order[at];
|
|
281
|
+
if (compareEntries(blob, offsets, left, right) >= 0) {
|
|
282
|
+
corrupt(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
function requireSpellingEntriesAscend(blob, offsets, tokens, name) {
|
|
287
|
+
for (let at = 1; at < tokens.length; at++) {
|
|
288
|
+
const order = compareEntries(blob, offsets, at - 1, at) || tokens[at - 1] - tokens[at];
|
|
289
|
+
if (order >= 0) corrupt(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function requireBelow(values, limit, name) {
|
|
293
|
+
for (let at = 0; at < values.length; at++) {
|
|
294
|
+
if (values[at] >= limit)
|
|
295
|
+
corrupt(`${name} holds ${values[at]} at entry ${at}, at or above ${limit}`, { field: name, at });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
function requireNonZero(values, name) {
|
|
299
|
+
for (let at = 0; at < values.length; at++) {
|
|
300
|
+
if (values[at] === 0) corrupt(`${name} holds 0 at entry ${at}`, { field: name, at });
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
function requirePermutation(order, name) {
|
|
304
|
+
const seen = new Uint8Array(order.length);
|
|
305
|
+
for (let at = 0; at < order.length; at++) {
|
|
306
|
+
const ordinal = order[at];
|
|
307
|
+
if (ordinal >= order.length || seen[ordinal] === 1) {
|
|
308
|
+
corrupt(`${name} repeats or overruns ordinal ${ordinal}`, { field: name, at });
|
|
309
|
+
}
|
|
310
|
+
seen[ordinal] = 1;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
function requireValuesAscendWithoutNaN(values, name) {
|
|
314
|
+
for (let at = 0; at < values.length; at++) {
|
|
315
|
+
if (Number.isNaN(values[at])) corrupt(`${name} holds NaN at entry ${at}`, { field: name, at });
|
|
316
|
+
if (at > 0 && values[at] < values[at - 1]) corrupt(`${name} descends at entry ${at}`, { field: name, at });
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
function requireWithin(values, lowest, highest, name) {
|
|
320
|
+
for (let at = 0; at < values.length; at++) {
|
|
321
|
+
const value = values[at];
|
|
322
|
+
if (!(value >= lowest && value <= highest)) {
|
|
323
|
+
corrupt(`${name} holds ${value} at entry ${at}, outside ${lowest} to ${highest}`, { field: name, at });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
function requireRunsAscend(runs, name) {
|
|
328
|
+
for (let run = 1; run * 3 < runs.length; run++) {
|
|
329
|
+
const at = run * 3;
|
|
330
|
+
const before = at - 3;
|
|
331
|
+
let diff = runs[at] - runs[before];
|
|
332
|
+
if (diff === 0) diff = runs[at + 1] - runs[before + 1];
|
|
333
|
+
if (diff === 0) diff = runs[at + 2] - runs[before + 2];
|
|
334
|
+
if (diff <= 0) corrupt(`${name} fails to ascend strictly at run ${run}`, { field: name, run });
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// src/serialization/keyword-segment/decode-columns.ts
|
|
339
|
+
var LATITUDE_LIMIT = 90;
|
|
340
|
+
var LONGITUDE_LIMIT = 180;
|
|
341
|
+
var CODE_POINTS_PER_RUN = 3;
|
|
342
|
+
function uint32Column(raw, key, name, context) {
|
|
343
|
+
return uint32sFrom(requireBytes(raw[key], `${name}.${key}`, 4), context.shared);
|
|
344
|
+
}
|
|
345
|
+
function ordinalColumn(raw, key, name, context) {
|
|
346
|
+
const values = uint32Column(raw, key, name, context);
|
|
347
|
+
requireOrdinals(values, context.documentCount, `${name}.${key}`);
|
|
348
|
+
return values;
|
|
349
|
+
}
|
|
350
|
+
function eachColumn(raw, name) {
|
|
351
|
+
return requireList(raw, name).map((entry, at) => {
|
|
352
|
+
const column = requireRecord(entry, `${name}[${at}]`);
|
|
353
|
+
return { column, label: `${name}[${at}]` };
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
function decodeFieldLengths(raw, context) {
|
|
357
|
+
const fieldLengthNames = [];
|
|
358
|
+
const fieldLengthColumns = [];
|
|
359
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
360
|
+
for (const { column, label } of eachColumn(raw, "fieldLengths")) {
|
|
361
|
+
const field = requireString(column.field, `${label}.field`);
|
|
362
|
+
const lengths = uint32Column(column, "lengths", label, context);
|
|
363
|
+
requireCount(lengths.length, context.documentCount, `${label}.lengths`);
|
|
364
|
+
let total = 0;
|
|
365
|
+
for (let at = 0; at < lengths.length; at++) total += lengths[at];
|
|
366
|
+
fieldLengthNames.push(field);
|
|
367
|
+
fieldLengthColumns.push(lengths);
|
|
368
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + total;
|
|
369
|
+
}
|
|
370
|
+
return { fieldLengthNames, fieldLengthColumns, totalFieldLengths };
|
|
371
|
+
}
|
|
372
|
+
function decodeValueColumns(raw, name, context) {
|
|
373
|
+
return eachColumn(raw, name).map(({ column, label }) => {
|
|
374
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
375
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
376
|
+
const values = float64sFrom(requireBytes(column.values, `${label}.values`, 8), context.shared);
|
|
377
|
+
requireCount(values.length, docIds.length, `${label}.values`);
|
|
378
|
+
requireValuesAscendWithoutNaN(values, `${label}.values`);
|
|
379
|
+
return { fieldPath, docIds, values };
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
function decodeBooleanColumns(raw, context) {
|
|
383
|
+
return eachColumn(raw, "boolean").map(({ column, label }) => ({
|
|
384
|
+
fieldPath: requireString(column.field, `${label}.field`),
|
|
385
|
+
trueDocs: ordinalColumn(column, "trueDocuments", label, context),
|
|
386
|
+
falseDocs: ordinalColumn(column, "falseDocuments", label, context)
|
|
387
|
+
}));
|
|
388
|
+
}
|
|
389
|
+
function decodeEnumColumns(raw, context) {
|
|
390
|
+
return eachColumn(raw, "enum").map(({ column, label }) => {
|
|
391
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
392
|
+
const values = requireStrings(column.values, `${label}.values`);
|
|
393
|
+
if (new Set(values).size !== values.length) corrupt(`${label}.values names a value twice`, { field: label });
|
|
394
|
+
const offsets = uint32Column(column, "offsets", label, context);
|
|
395
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
396
|
+
requireCount(offsets.length, values.length + 1, `${label}.offsets`);
|
|
397
|
+
requireOffsets(offsets, docIds.length, `${label}.offsets`);
|
|
398
|
+
return { fieldPath, values, offsets, docIds };
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
function decodeGeopointColumns(raw, context) {
|
|
402
|
+
return eachColumn(raw, "geopoint").map(({ column, label }) => {
|
|
403
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
404
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
405
|
+
const latitudes = float64sFrom(requireBytes(column.latitudes, `${label}.latitudes`, 8), context.shared);
|
|
406
|
+
const longitudes = float64sFrom(requireBytes(column.longitudes, `${label}.longitudes`, 8), context.shared);
|
|
407
|
+
requireCount(latitudes.length, docIds.length, `${label}.latitudes`);
|
|
408
|
+
requireCount(longitudes.length, docIds.length, `${label}.longitudes`);
|
|
409
|
+
requireWithin(latitudes, -LATITUDE_LIMIT, LATITUDE_LIMIT, `${label}.latitudes`);
|
|
410
|
+
requireWithin(longitudes, -LONGITUDE_LIMIT, LONGITUDE_LIMIT, `${label}.longitudes`);
|
|
411
|
+
return { fieldPath, docIds, latitudes, longitudes };
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
function decodePatternColumns(raw, context) {
|
|
415
|
+
return eachColumn(raw, "pattern").map(({ column, label }) => {
|
|
416
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
417
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
418
|
+
requireStrictlyAscending(docIds, 0, docIds.length, `${label}.documents`);
|
|
419
|
+
const runs = uint32sFrom(requireBytes(column.runs, `${label}.runs`, 4 * CODE_POINTS_PER_RUN), context.shared);
|
|
420
|
+
requireRunsAscend(runs, `${label}.runs`);
|
|
421
|
+
const offsets = uint32Column(column, "offsets", label, context);
|
|
422
|
+
const postings = ordinalColumn(column, "postings", label, context);
|
|
423
|
+
requireCount(offsets.length, runs.length / CODE_POINTS_PER_RUN + 1, `${label}.offsets`);
|
|
424
|
+
requireOffsets(offsets, postings.length, `${label}.offsets`);
|
|
425
|
+
for (let run = 0; run + 1 < offsets.length; run++) {
|
|
426
|
+
requireStrictlyAscending(postings, offsets[run], offsets[run + 1], `${label}.postings`);
|
|
427
|
+
}
|
|
428
|
+
return { fieldPath, docIds, runs, offsets, postings };
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
function decodeSurfaceForms(raw, context) {
|
|
432
|
+
if (raw === null || raw === void 0) return null;
|
|
433
|
+
const column = requireRecord(raw, "surfaceForms");
|
|
434
|
+
const blob = bytesFrom(requireBytes(column.text, "surfaceForms.text", 1), context.shared);
|
|
435
|
+
const offsets = uint32Column(column, "offsets", "surfaceForms", context);
|
|
436
|
+
const counts = uint32Column(column, "counts", "surfaceForms", context);
|
|
437
|
+
requireCount(offsets.length, counts.length * 2 + 1, "surfaceForms.offsets");
|
|
438
|
+
requireOffsets(offsets, blob.length, "surfaceForms.offsets");
|
|
439
|
+
requireUtf8Entries(blob, offsets, "surfaceForms.text");
|
|
440
|
+
return counts.length === 0 ? null : { blob, offsets, counts };
|
|
441
|
+
}
|
|
442
|
+
function decodePartialWords(raw, tokenCount, context) {
|
|
443
|
+
if (raw === null || raw === void 0) return null;
|
|
444
|
+
const column = requireRecord(raw, "partialWords");
|
|
445
|
+
const fields = requireStrings(column.fields, "partialWords.fields");
|
|
446
|
+
const spellings = bytesFrom(requireBytes(column.text, "partialWords.text", 1), context.shared);
|
|
447
|
+
const spellingOffsets = uint32Column(column, "offsets", "partialWords", context);
|
|
448
|
+
const spellingTokens = uint32Column(column, "tokens", "partialWords", context);
|
|
449
|
+
const spellingCounts = uint32Column(column, "counts", "partialWords", context);
|
|
450
|
+
const entries = spellingTokens.length;
|
|
451
|
+
requireCount(spellingOffsets.length, entries + 1, "partialWords.offsets");
|
|
452
|
+
requireCount(spellingCounts.length, entries, "partialWords.counts");
|
|
453
|
+
requireOffsets(spellingOffsets, spellings.length, "partialWords.offsets");
|
|
454
|
+
requireUtf8Entries(spellings, spellingOffsets, "partialWords.text");
|
|
455
|
+
requireBelow(spellingTokens, tokenCount, "partialWords.tokens");
|
|
456
|
+
requireNonZero(spellingCounts, "partialWords.counts");
|
|
457
|
+
requireSpellingEntriesAscend(spellings, spellingOffsets, spellingTokens, "partialWords");
|
|
458
|
+
const runs = uint32sFrom(requireBytes(column.runs, "partialWords.runs", 4 * CODE_POINTS_PER_RUN), context.shared);
|
|
459
|
+
requireRunsAscend(runs, "partialWords.runs");
|
|
460
|
+
const runOffsets = uint32Column(column, "runOffsets", "partialWords", context);
|
|
461
|
+
const runEntries = uint32Column(column, "runEntries", "partialWords", context);
|
|
462
|
+
requireCount(runOffsets.length, runs.length / CODE_POINTS_PER_RUN + 1, "partialWords.runOffsets");
|
|
463
|
+
requireOffsets(runOffsets, runEntries.length, "partialWords.runOffsets");
|
|
464
|
+
requireBelow(runEntries, entries, "partialWords.runEntries");
|
|
465
|
+
for (let run = 0; run + 1 < runOffsets.length; run++) {
|
|
466
|
+
requireStrictlyAscending(runEntries, runOffsets[run], runOffsets[run + 1], "partialWords.runEntries");
|
|
467
|
+
}
|
|
468
|
+
return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// src/serialization/keyword-segment/write-records.ts
|
|
472
|
+
var UINT64_BYTES = 8;
|
|
473
|
+
function uint64Bytes(values) {
|
|
474
|
+
const bytes = new Uint8Array(values.length * UINT64_BYTES);
|
|
475
|
+
const view = new DataView(bytes.buffer);
|
|
476
|
+
for (let at = 0; at < values.length; at++) view.setBigUint64(at * UINT64_BYTES, BigInt(values[at]), true);
|
|
477
|
+
return bytes;
|
|
478
|
+
}
|
|
479
|
+
function uint64sOf(bytes, name) {
|
|
480
|
+
const values = new Float64Array(bytes.byteLength / UINT64_BYTES);
|
|
481
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
482
|
+
for (let at = 0; at < values.length; at++) {
|
|
483
|
+
const value = view.getBigUint64(at * UINT64_BYTES, true);
|
|
484
|
+
if (value > BigInt(Number.MAX_SAFE_INTEGER)) corrupt(`${name} holds a value above 2^53 - 1`, { at });
|
|
485
|
+
values[at] = Number(value);
|
|
486
|
+
}
|
|
487
|
+
return values;
|
|
488
|
+
}
|
|
489
|
+
function decodeWriteRecords(raw, documentCount) {
|
|
490
|
+
const rawSeqNos = requireBytesOrNil(raw.seqNos, "seqNos", UINT64_BYTES);
|
|
491
|
+
const rawTerms = requireBytesOrNil(raw.primaryTerms, "primaryTerms", UINT64_BYTES);
|
|
492
|
+
if (rawSeqNos === null !== (rawTerms === null))
|
|
493
|
+
corrupt("seqNos and primaryTerms must both be present or both absent");
|
|
494
|
+
if (rawSeqNos === null || rawTerms === null) return null;
|
|
495
|
+
const seqNos = uint64sOf(rawSeqNos, "seqNos");
|
|
496
|
+
const primaryTerms = uint64sOf(rawTerms, "primaryTerms");
|
|
497
|
+
requireCount(seqNos.length, documentCount, "seqNos");
|
|
498
|
+
requireCount(primaryTerms.length, documentCount, "primaryTerms");
|
|
499
|
+
for (let at = 0; at < documentCount; at++) {
|
|
500
|
+
if (seqNos[at] === 0 && primaryTerms[at] !== 0) {
|
|
501
|
+
corrupt(`primaryTerms gives document ${at} a term without a seqNo`, { at });
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
return { seqNos, primaryTerms };
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// src/serialization/keyword-segment/decode.ts
|
|
508
|
+
function decodeSchema(raw) {
|
|
509
|
+
const schema = requireRecord(raw, "schema");
|
|
510
|
+
requireValidFieldTypes(schema);
|
|
511
|
+
return schema;
|
|
512
|
+
}
|
|
513
|
+
function decodeIds(raw, context) {
|
|
514
|
+
const blob = bytesFrom(requireBytes(raw.ids, "ids", 1), context.shared);
|
|
515
|
+
const offsets = uint32sFrom(requireBytes(raw.idOffsets, "idOffsets", 4), context.shared);
|
|
516
|
+
const sortedOrdinals = uint32sFrom(requireBytes(raw.idOrder, "idOrder", 4), context.shared);
|
|
517
|
+
requireCount(offsets.length, context.documentCount + 1, "idOffsets");
|
|
518
|
+
requireCount(sortedOrdinals.length, context.documentCount, "idOrder");
|
|
519
|
+
requireOffsets(offsets, blob.length, "idOffsets");
|
|
520
|
+
requireUtf8Entries(blob, offsets, "ids");
|
|
521
|
+
requirePermutation(sortedOrdinals, "idOrder");
|
|
522
|
+
requireEntriesAscend(blob, offsets, sortedOrdinals, "ids");
|
|
523
|
+
return { blob, offsets, sortedOrdinals };
|
|
524
|
+
}
|
|
525
|
+
function vectorFieldPathsOf(schema) {
|
|
526
|
+
return Object.keys(schema).filter((fieldPath) => VECTOR_PATTERN.test(schema[fieldPath]));
|
|
527
|
+
}
|
|
528
|
+
function decodeDocuments(raw, schema, context) {
|
|
529
|
+
const blob = bytesFrom(requireBytes(raw.documents, "documents", 1), context.shared);
|
|
530
|
+
const offsets = uint32sFrom(requireBytes(raw.documentOffsets, "documentOffsets", 4), context.shared);
|
|
531
|
+
requireCount(offsets.length, context.documentCount + 1, "documentOffsets");
|
|
532
|
+
requireOffsets(offsets, blob.length, "documentOffsets");
|
|
533
|
+
const discardedFields = vectorFieldPathsOf(schema);
|
|
534
|
+
return discardedFields.length === 0 ? { blob, offsets } : { blob, offsets, discardedFields };
|
|
535
|
+
}
|
|
536
|
+
function identitySlots(count, shared) {
|
|
537
|
+
const slots = uint32sFrom(new Uint8Array(count * 4), shared);
|
|
538
|
+
for (let at = 0; at < count; at++) slots[at] = at;
|
|
539
|
+
return slots;
|
|
540
|
+
}
|
|
541
|
+
function requireDocumentFrequencies(frequencies, postingOffsets) {
|
|
542
|
+
for (let token = 0; token < frequencies.length; token++) {
|
|
543
|
+
const postings = postingOffsets[token + 1] - postingOffsets[token];
|
|
544
|
+
if (frequencies[token] === 0 || frequencies[token] > postings) {
|
|
545
|
+
corrupt(`documentFrequencies gives token ${token} a count of ${frequencies[token]} over ${postings} postings`, {
|
|
546
|
+
token
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
function requireFieldNumbers(fields, fieldNames) {
|
|
552
|
+
for (let at = 0; at < fields.length; at++) {
|
|
553
|
+
if (fields[at] >= fieldNames) corrupt(`postingFields names field ${fields[at]} of ${fieldNames}`, { at });
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
function decodePostings(raw, fieldNames, context) {
|
|
557
|
+
const blob = bytesFrom(requireBytes(raw.tokens, "tokens", 1), context.shared);
|
|
558
|
+
const offsets = uint32sFrom(requireBytes(raw.tokenOffsets, "tokenOffsets", 4), context.shared);
|
|
559
|
+
requireOffsets(offsets, blob.length, "tokenOffsets");
|
|
560
|
+
requireUtf8Entries(blob, offsets, "tokens");
|
|
561
|
+
requireEntriesAscend(blob, offsets, null, "tokens");
|
|
562
|
+
const tokens = offsets.length - 1;
|
|
563
|
+
const documentFrequencies = uint32sFrom(
|
|
564
|
+
requireBytes(raw.documentFrequencies, "documentFrequencies", 4),
|
|
565
|
+
context.shared
|
|
566
|
+
);
|
|
567
|
+
const postingOffsets = uint32sFrom(requireBytes(raw.postingOffsets, "postingOffsets", 4), context.shared);
|
|
568
|
+
const postingDocIds = uint32sFrom(requireBytes(raw.postingDocuments, "postingDocuments", 4), context.shared);
|
|
569
|
+
const postingFrequencies = uint16sFrom(requireBytes(raw.postingFrequencies, "postingFrequencies", 2), context.shared);
|
|
570
|
+
const postingFieldIndices = uint16sFrom(requireBytes(raw.postingFields, "postingFields", 2), context.shared);
|
|
571
|
+
requireCount(documentFrequencies.length, tokens, "documentFrequencies");
|
|
572
|
+
requireCount(postingOffsets.length, tokens + 1, "postingOffsets");
|
|
573
|
+
requireOffsets(postingOffsets, postingDocIds.length, "postingOffsets");
|
|
574
|
+
requireCount(postingFrequencies.length, postingDocIds.length, "postingFrequencies");
|
|
575
|
+
requireCount(postingFieldIndices.length, postingDocIds.length, "postingFields");
|
|
576
|
+
requireOrdinals(postingDocIds, context.documentCount, "postingDocuments");
|
|
577
|
+
requireFieldNumbers(postingFieldIndices, fieldNames.length);
|
|
578
|
+
requireDocumentFrequencies(documentFrequencies, postingOffsets);
|
|
579
|
+
const rawPositionOffsets = requireBytesOrNil(raw.positionOffsets, "positionOffsets", 4);
|
|
580
|
+
const rawPositions = requireBytesOrNil(raw.positions, "positions", 4);
|
|
581
|
+
if (rawPositionOffsets === null !== (rawPositions === null)) {
|
|
582
|
+
corrupt("positionOffsets and positions must both be nil or both hold bytes");
|
|
583
|
+
}
|
|
584
|
+
let positionOffsets = null;
|
|
585
|
+
let positionValues = null;
|
|
586
|
+
if (rawPositionOffsets !== null && rawPositions !== null) {
|
|
587
|
+
positionOffsets = uint32sFrom(rawPositionOffsets, context.shared);
|
|
588
|
+
positionValues = uint32sFrom(rawPositions, context.shared);
|
|
589
|
+
requireCount(positionOffsets.length, postingDocIds.length + 1, "positionOffsets");
|
|
590
|
+
requireOffsets(positionOffsets, positionValues.length, "positionOffsets");
|
|
591
|
+
}
|
|
592
|
+
return {
|
|
593
|
+
tokenTable: { blob, offsets, payloadSlots: identitySlots(tokens, context.shared), documentFrequencies },
|
|
594
|
+
postingOffsets,
|
|
595
|
+
postingDocIds,
|
|
596
|
+
postingFrequencies,
|
|
597
|
+
postingFieldIndices,
|
|
598
|
+
positionOffsets,
|
|
599
|
+
positionValues
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
function decodeKeywordSegment(payload, shared) {
|
|
603
|
+
let decoded;
|
|
604
|
+
try {
|
|
605
|
+
decoded = decode(payload);
|
|
606
|
+
} catch (error) {
|
|
607
|
+
corrupt(`payload is not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
|
|
608
|
+
}
|
|
609
|
+
const raw = requireRecord(decoded, "payload");
|
|
610
|
+
if (raw.v !== KEYWORD_SEGMENT_PAYLOAD_VERSION) {
|
|
611
|
+
corrupt(`payload version ${String(raw.v)} is not ${KEYWORD_SEGMENT_PAYLOAD_VERSION}`, { version: raw.v });
|
|
612
|
+
}
|
|
613
|
+
const schema = decodeSchema(raw.schema);
|
|
614
|
+
const tombstones = requireStrings(raw.tombstones, "tombstones");
|
|
615
|
+
const documentCount = requireUint32(raw.documentCount, "documentCount");
|
|
616
|
+
const fieldNames = requireStrings(raw.fieldNames, "fieldNames");
|
|
617
|
+
if (fieldNames.length > MAX_SEGMENT_FIELD_NAMES) {
|
|
618
|
+
corrupt(`fieldNames holds ${fieldNames.length} names, above the limit of ${MAX_SEGMENT_FIELD_NAMES}`);
|
|
619
|
+
}
|
|
620
|
+
const context = { documentCount, shared };
|
|
621
|
+
const postings = decodePostings(raw, fieldNames, context);
|
|
622
|
+
const arrays = {
|
|
623
|
+
documentCount,
|
|
624
|
+
fieldNames,
|
|
625
|
+
idTable: decodeIds(raw, context),
|
|
626
|
+
documentTable: decodeDocuments(raw, schema, context),
|
|
627
|
+
...postings,
|
|
628
|
+
...decodeFieldLengths(raw.fieldLengths, context),
|
|
629
|
+
numeric: decodeValueColumns(raw.numeric, "numeric", context),
|
|
630
|
+
sizes: decodeValueColumns(raw.sizes, "sizes", context),
|
|
631
|
+
boolean: decodeBooleanColumns(raw.boolean, context),
|
|
632
|
+
enums: decodeEnumColumns(raw.enum, context),
|
|
633
|
+
geo: decodeGeopointColumns(raw.geopoint, context),
|
|
634
|
+
pattern: decodePatternColumns(raw.pattern, context),
|
|
635
|
+
surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
|
|
636
|
+
partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
|
|
637
|
+
};
|
|
638
|
+
const writeRecords = decodeWriteRecords(raw, documentCount);
|
|
639
|
+
return writeRecords === null ? { schema, tombstones, arrays } : { schema, tombstones, arrays, writeRecords };
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
export { KEYWORD_SEGMENT_ENVELOPE_VERSION, KEYWORD_SEGMENT_PAYLOAD_VERSION, applyDeleteEntry, applyIndexEntry, decodeBooleanColumns, decodeEnumColumns, decodeFieldLengths, decodeGeopointColumns, decodeKeywordSegment, decodePartialWords, decodePatternColumns, decodeSurfaceForms, decodeValueColumns, littleEndianBytes, requireEntriesAscend, requireOffsets, requireUtf8Entries, restoreVectorFields, setNestedValue, uint32sFrom, uint64Bytes };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// src/search/constants.ts
|
|
2
|
+
var RESULT_WINDOW = 1e4;
|
|
3
|
+
var DEFAULT_PAGE_SIZE = 10;
|
|
4
|
+
var BM25_DISPLAYED_NAME_LENGTH = 64;
|
|
5
|
+
var MAX_CURSOR_LENGTH = 40960;
|
|
6
|
+
var MAX_SORT_FIELDS = 8;
|
|
7
|
+
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
8
|
+
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
9
|
+
var PREFIX_MAX_EXPANSIONS = 50;
|
|
10
|
+
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
11
|
+
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
12
|
+
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
13
|
+
var MAX_TOLERANCE = 10;
|
|
14
|
+
var MAX_PREFIX_LENGTH = 1024;
|
|
15
|
+
var MAX_PHRASE_WORDS = 32;
|
|
16
|
+
var MAX_QUERY_PHRASES = 32;
|
|
17
|
+
var DEFAULT_MAX_PER_GROUP = 1;
|
|
18
|
+
var DEFAULT_RANK_CONSTANT = 60;
|
|
19
|
+
var MIN_RANK_CONSTANT = 1;
|
|
20
|
+
var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
21
|
+
var MIN_LINEAR_ALPHA = 0;
|
|
22
|
+
var MAX_LINEAR_ALPHA = 1;
|
|
23
|
+
var MIN_HYBRID_DEPTH = 1;
|
|
24
|
+
var MAX_HYBRID_DEPTH = 1e4;
|
|
25
|
+
var HYBRID_RESCORE_WINDOW = 10;
|
|
26
|
+
|
|
27
|
+
export { BM25_DISPLAYED_NAME_LENGTH, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, HYBRID_RESCORE_WINDOW, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_HYBRID_DEPTH, MAX_LINEAR_ALPHA, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|