@delali/narsil 0.3.1-next.157 → 0.3.1-next.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
- package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +66 -18
- package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1199 -1092
- package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -16
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -227
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
- package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +95 -52
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +33 -13
- package/dist/server.mjs +81 -100
- package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +88 -72
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +89 -59
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +66 -50
- package/package.json +233 -8
- package/dist/analysis-change-CKEMM2MC.mjs +0 -55
- package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
- package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-25MTVDYD.mjs +0 -432
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-AI5UI46D.mjs +0 -12
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
- package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
- package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
- package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MY2XKH22.mjs +0 -40
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
- package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
- package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
- package/dist/browser/chunk-V43N6TZO.mjs +0 -857
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
- package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-5ESVMFDJ.mjs +0 -20
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NDSQLPMQ.mjs +0 -25
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-HYXPVJVS.mjs +0 -32
- package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
- package/dist/browser/retention-TY3L3BFN.mjs +0 -211
- package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
- package/dist/browser/start-2UZTOBG5.mjs +0 -26
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-ULXAN5B4.mjs +0 -196
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-2ROUFMQX.mjs +0 -543
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-4B7JIGQD.mjs +0 -106
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BNL5OPRV.mjs +0 -634
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-C7XQYG6U.mjs +0 -50
- package/dist/chunk-CQZXZ75V.mjs +0 -562
- package/dist/chunk-CWS55TEX.mjs +0 -242
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FEZVDOWX.mjs +0 -6996
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GPWMUPCA.mjs +0 -1126
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JSV6OQ4C.mjs +0 -74
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KCQYFO4K.mjs +0 -62
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LDNCLRZU.mjs +0 -218
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MW6BR4QP.mjs +0 -316
- package/dist/chunk-MWD5VPN7.mjs +0 -21
- package/dist/chunk-ODYKO5TO.mjs +0 -101
- package/dist/chunk-OG7UGJ6F.mjs +0 -63
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-OYABFKTW.mjs +0 -1974
- package/dist/chunk-P5LANSH6.mjs +0 -121
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PIH5SJ4N.mjs +0 -7481
- package/dist/chunk-PSZI4O24.mjs +0 -253
- package/dist/chunk-PV2CG7EJ.mjs +0 -886
- package/dist/chunk-PWGYQXML.mjs +0 -470
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QLDABJZM.mjs +0 -20
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RCX4WNLC.mjs +0 -151
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RIHIVFIQ.mjs +0 -240
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RWURUDSP.mjs +0 -2639
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SOI6LTHM.mjs +0 -216
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UD5HIS4U.mjs +0 -382
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-UJDJB3NE.mjs +0 -913
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VSLSGASN.mjs +0 -30
- package/dist/chunk-VZ2ONHEE.mjs +0 -3715
- package/dist/chunk-W7OSSKH5.mjs +0 -1427
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-V5DRIECI.mjs +0 -34
- package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
- package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-GPKLTSKA.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-JYHFTVPS.mjs +0 -63
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
- package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
- package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
- package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
- package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
- package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
- package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
- package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
- package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
- package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
- package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
- package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
- package/dist/release-SMR4WJD4.mjs +0 -32
- package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
- package/dist/restore-queue-LZH7LMLR.mjs +0 -55
- package/dist/retention-3K7ARGAG.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-O7ICP7NJ.mjs +0 -783
- package/dist/start-S4PFP6RS.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-YLJ3ZPQ3.mjs +0 -50
- package/dist/write-2OBJGLFS.mjs +0 -233
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
package/dist/chunk-W7OSSKH5.mjs
DELETED
|
@@ -1,1427 +0,0 @@
|
|
|
1
|
-
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-P5LANSH6.mjs';
|
|
2
|
-
import { defaultQuantizationFor } from './chunk-MATMVH65.mjs';
|
|
3
|
-
import { DEFAULT_VECTOR_METRIC } from './chunk-VD7J552I.mjs';
|
|
4
|
-
import { gunzip, gzip } from './chunk-ZXNNR73R.mjs';
|
|
5
|
-
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-WZ7YV54N.mjs';
|
|
6
|
-
import { decodeSegmentManifest } from './chunk-CWS55TEX.mjs';
|
|
7
|
-
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
8
|
-
import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-BNL5OPRV.mjs';
|
|
9
|
-
import { manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-JTYPJF3T.mjs';
|
|
10
|
-
import { storedStateToWire, storedStateFromWire } from './chunk-ODYKO5TO.mjs';
|
|
11
|
-
import { VERSION, rulesOfStoredSynonyms, engineVersionParts, packChecksummedEnvelope, packEnvelopeWithChecksum } from './chunk-FEZVDOWX.mjs';
|
|
12
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-VZ2ONHEE.mjs';
|
|
13
|
-
import { readHeader, HEADER_SIZE, writeHeader, checksumOfChunks } from './chunk-3OH67G7R.mjs';
|
|
14
|
-
import { crc32 } from './chunk-7GXJUJ24.mjs';
|
|
15
|
-
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-W5EB5XKI.mjs';
|
|
16
|
-
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-FB6463IN.mjs';
|
|
17
|
-
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
18
|
-
import { VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
19
|
-
import { NarsilError, ErrorCodes } from './chunk-MP7ABPTD.mjs';
|
|
20
|
-
import { compareCodePoints } from './chunk-TBC3XUL2.mjs';
|
|
21
|
-
import { decode, encode } from '@msgpack/msgpack';
|
|
22
|
-
|
|
23
|
-
// src/engine/durability-metadata.ts
|
|
24
|
-
var VECTOR_TYPE = /^vector\[(\d+)\]$/;
|
|
25
|
-
function vectorFieldsOf(schema, promotion) {
|
|
26
|
-
const fields = {};
|
|
27
|
-
for (const [fieldPath, type] of Object.entries(schema)) {
|
|
28
|
-
const match = VECTOR_TYPE.exec(type);
|
|
29
|
-
if (match === null) continue;
|
|
30
|
-
const dimension = Number(match[1]);
|
|
31
|
-
fields[fieldPath] = {
|
|
32
|
-
dimension,
|
|
33
|
-
metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
|
|
34
|
-
quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
return Object.keys(fields).length === 0 ? null : fields;
|
|
38
|
-
}
|
|
39
|
-
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
40
|
-
const metadata = {
|
|
41
|
-
indexName,
|
|
42
|
-
schema: config.schema,
|
|
43
|
-
language: config.language,
|
|
44
|
-
partitionCount: partitionCount ?? config.partitionCount,
|
|
45
|
-
bm25Params: {
|
|
46
|
-
k1: config.k1,
|
|
47
|
-
b: config.b,
|
|
48
|
-
...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
|
|
49
|
-
},
|
|
50
|
-
createdAt: Date.now(),
|
|
51
|
-
engineVersion: VERSION
|
|
52
|
-
};
|
|
53
|
-
metadata.documentCount = documentCount ?? config.documentCount;
|
|
54
|
-
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
55
|
-
if (vectorFields !== null) {
|
|
56
|
-
metadata.vectorFields = vectorFields;
|
|
57
|
-
}
|
|
58
|
-
if (config.embedding) {
|
|
59
|
-
metadata.embedding = config.embedding;
|
|
60
|
-
}
|
|
61
|
-
if (config.surfaceForms !== void 0) {
|
|
62
|
-
metadata.surfaceForms = config.surfaceForms;
|
|
63
|
-
}
|
|
64
|
-
if (config.analysisRevision !== void 0) {
|
|
65
|
-
metadata.analysisRevision = config.analysisRevision;
|
|
66
|
-
}
|
|
67
|
-
if (config.arrayPositionGap !== void 0) {
|
|
68
|
-
metadata.arrayPositionGap = config.arrayPositionGap;
|
|
69
|
-
}
|
|
70
|
-
if (config.tokenizer !== void 0) {
|
|
71
|
-
metadata.tokenizer = config.tokenizer;
|
|
72
|
-
}
|
|
73
|
-
if (config.stopWords !== void 0) {
|
|
74
|
-
metadata.stopWords = config.stopWords;
|
|
75
|
-
}
|
|
76
|
-
if (config.stopWordList !== void 0) {
|
|
77
|
-
metadata.stopWordList = config.stopWordList;
|
|
78
|
-
}
|
|
79
|
-
if (config.analysisGeneration !== void 0) {
|
|
80
|
-
metadata.analysisGeneration = config.analysisGeneration;
|
|
81
|
-
}
|
|
82
|
-
if (config.pendingAnalysis !== void 0) {
|
|
83
|
-
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
84
|
-
}
|
|
85
|
-
if (config.partitionLimits !== void 0) {
|
|
86
|
-
metadata.partitionLimits = config.partitionLimits;
|
|
87
|
-
}
|
|
88
|
-
if (config.defaultScoring !== void 0) {
|
|
89
|
-
metadata.defaultScoring = config.defaultScoring;
|
|
90
|
-
}
|
|
91
|
-
if (config.trackPositions !== void 0) {
|
|
92
|
-
metadata.trackPositions = config.trackPositions;
|
|
93
|
-
}
|
|
94
|
-
if (config.textStorage !== void 0) {
|
|
95
|
-
metadata.textStorage = config.textStorage;
|
|
96
|
-
}
|
|
97
|
-
if (config.strict !== void 0) {
|
|
98
|
-
metadata.strict = config.strict;
|
|
99
|
-
}
|
|
100
|
-
if (config.required !== void 0) {
|
|
101
|
-
metadata.required = config.required;
|
|
102
|
-
}
|
|
103
|
-
if (config.vectorPromotion !== void 0) {
|
|
104
|
-
metadata.vectorPromotion = config.vectorPromotion;
|
|
105
|
-
}
|
|
106
|
-
if (config.patternValueLimit !== void 0) {
|
|
107
|
-
metadata.patternValueLimit = config.patternValueLimit;
|
|
108
|
-
}
|
|
109
|
-
if (config.indexUuid !== void 0) {
|
|
110
|
-
metadata.indexUuid = config.indexUuid;
|
|
111
|
-
}
|
|
112
|
-
if (config.synonyms !== void 0) {
|
|
113
|
-
metadata.synonyms = config.synonyms;
|
|
114
|
-
}
|
|
115
|
-
if (config.heldPartitions !== void 0) {
|
|
116
|
-
metadata.heldPartitions = config.heldPartitions;
|
|
117
|
-
}
|
|
118
|
-
if (config.sync !== void 0) metadata.sync = config.sync;
|
|
119
|
-
if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
|
|
120
|
-
if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
|
|
121
|
-
if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
|
|
122
|
-
return metadata;
|
|
123
|
-
}
|
|
124
|
-
function metadataConfigOf(entry) {
|
|
125
|
-
const embedding = entry.config.embedding ? {
|
|
126
|
-
fields: entry.config.embedding.fields,
|
|
127
|
-
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
128
|
-
} : void 0;
|
|
129
|
-
return {
|
|
130
|
-
...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
|
|
131
|
-
...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
|
|
132
|
-
documentCount: entry.documentCount,
|
|
133
|
-
partitionCount: entry.partitionCount,
|
|
134
|
-
schema: flattenSchema(entry.config.schema),
|
|
135
|
-
language: entry.language.name,
|
|
136
|
-
k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
|
|
137
|
-
b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
|
|
138
|
-
...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
|
|
139
|
-
...embedding !== void 0 ? { embedding } : {},
|
|
140
|
-
surfaceForms: entry.config.surfaceForms !== false,
|
|
141
|
-
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
142
|
-
...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
|
|
143
|
-
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
144
|
-
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
145
|
-
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
146
|
-
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
147
|
-
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
148
|
-
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
149
|
-
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
150
|
-
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
151
|
-
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
152
|
-
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
153
|
-
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
154
|
-
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
155
|
-
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
|
|
156
|
-
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
|
|
157
|
-
...syncMetadataOf(entry),
|
|
158
|
-
...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
function syncMetadataOf(entry) {
|
|
162
|
-
const sync = entry.sync;
|
|
163
|
-
if (sync === null) return {};
|
|
164
|
-
return {
|
|
165
|
-
sync: {
|
|
166
|
-
enabled: true,
|
|
167
|
-
...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
|
|
168
|
-
...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
|
|
169
|
-
},
|
|
170
|
-
syncActive: sync.active,
|
|
171
|
-
syncEpoch: sync.epoch
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
function durableMetadataConfigOf(entry) {
|
|
175
|
-
const terms = metadataConfigOf(entry);
|
|
176
|
-
const durable = entry.durableAnalysis;
|
|
177
|
-
if (durable === null) return terms;
|
|
178
|
-
const {
|
|
179
|
-
language,
|
|
180
|
-
tokenizer,
|
|
181
|
-
stopWords,
|
|
182
|
-
stopWordList,
|
|
183
|
-
analysisRevision: _analysisRevision,
|
|
184
|
-
arrayPositionGap: _arrayPositionGap,
|
|
185
|
-
analysisGeneration: _analysisGeneration,
|
|
186
|
-
...rest
|
|
187
|
-
} = terms;
|
|
188
|
-
return {
|
|
189
|
-
...rest,
|
|
190
|
-
language: durable.language,
|
|
191
|
-
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
192
|
-
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
193
|
-
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
194
|
-
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
195
|
-
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
196
|
-
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
197
|
-
pendingAnalysis: {
|
|
198
|
-
generation: entry.analysisGeneration,
|
|
199
|
-
language,
|
|
200
|
-
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
201
|
-
...stopWords !== void 0 ? { stopWords } : {},
|
|
202
|
-
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
203
|
-
}
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
207
|
-
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
// src/serialization/stored-vector-values.ts
|
|
211
|
-
function dropStoredVectorValues(documents, schema) {
|
|
212
|
-
const vectorFieldPaths = [];
|
|
213
|
-
for (const [fieldPath, fieldType] of Object.entries(schema)) {
|
|
214
|
-
if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
|
|
215
|
-
vectorFieldPaths.push(fieldPath);
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
if (vectorFieldPaths.length === 0) {
|
|
219
|
-
return;
|
|
220
|
-
}
|
|
221
|
-
for (const document of Object.values(documents)) {
|
|
222
|
-
for (const fieldPath of vectorFieldPaths) {
|
|
223
|
-
removeFieldAtPath(document.fields, fieldPath);
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
function removeFieldAtPath(fields, fieldPath) {
|
|
228
|
-
if (!fieldPath.includes(".")) {
|
|
229
|
-
delete fields[fieldPath];
|
|
230
|
-
return;
|
|
231
|
-
}
|
|
232
|
-
const segments = fieldPath.split(".");
|
|
233
|
-
let current = fields;
|
|
234
|
-
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
235
|
-
const next = current[segments[i]];
|
|
236
|
-
if (next === null || typeof next !== "object") {
|
|
237
|
-
return;
|
|
238
|
-
}
|
|
239
|
-
current = next;
|
|
240
|
-
}
|
|
241
|
-
delete current[segments[segments.length - 1]];
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// src/serialization/payload-v1/partition.ts
|
|
245
|
-
function sanitizeSurfaceForms(raw) {
|
|
246
|
-
if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
247
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
248
|
-
let count = 0;
|
|
249
|
-
for (const [surface, value] of Object.entries(raw)) {
|
|
250
|
-
if (typeof value === "number") {
|
|
251
|
-
result[surface] = value;
|
|
252
|
-
count++;
|
|
253
|
-
} else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
|
|
254
|
-
result[surface] = [value[0], value[1]];
|
|
255
|
-
count++;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
return count > 0 ? result : void 0;
|
|
259
|
-
}
|
|
260
|
-
function isWireRecord(value) {
|
|
261
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
262
|
-
}
|
|
263
|
-
function corruptPatternEntry(field) {
|
|
264
|
-
return new NarsilError(
|
|
265
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
266
|
-
`The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
|
|
267
|
-
{ field }
|
|
268
|
-
);
|
|
269
|
-
}
|
|
270
|
-
function patternIndexesFromWire(raw) {
|
|
271
|
-
if (raw === void 0 || raw === null) return {};
|
|
272
|
-
if (!isWireRecord(raw)) {
|
|
273
|
-
throw new NarsilError(
|
|
274
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
275
|
-
"The pattern entry of this partition must be a map of fields"
|
|
276
|
-
);
|
|
277
|
-
}
|
|
278
|
-
const pattern = /* @__PURE__ */ Object.create(null);
|
|
279
|
-
for (const [field, index] of Object.entries(raw)) {
|
|
280
|
-
if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
|
|
281
|
-
throw corruptPatternEntry(field);
|
|
282
|
-
}
|
|
283
|
-
const docIds = [];
|
|
284
|
-
for (const docId of index.doc_ids) {
|
|
285
|
-
if (typeof docId !== "string") throw corruptPatternEntry(field);
|
|
286
|
-
docIds.push(docId);
|
|
287
|
-
}
|
|
288
|
-
pattern[field] = { docIds, runs: index.runs };
|
|
289
|
-
}
|
|
290
|
-
return { pattern };
|
|
291
|
-
}
|
|
292
|
-
function validateHnswMetric(value) {
|
|
293
|
-
if (value === void 0 || value === null) return void 0;
|
|
294
|
-
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
295
|
-
return value;
|
|
296
|
-
}
|
|
297
|
-
return void 0;
|
|
298
|
-
}
|
|
299
|
-
function wireToPartition(raw) {
|
|
300
|
-
const documents = {};
|
|
301
|
-
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
302
|
-
documents[docId] = {
|
|
303
|
-
fields: doc.fields,
|
|
304
|
-
fieldLengths: doc.field_lengths ?? {}
|
|
305
|
-
};
|
|
306
|
-
}
|
|
307
|
-
requireValidFieldTypes(raw.schema ?? {});
|
|
308
|
-
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
309
|
-
const invertedIndex = {};
|
|
310
|
-
for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
|
|
311
|
-
invertedIndex[token] = {
|
|
312
|
-
docFrequency: list.doc_freq,
|
|
313
|
-
postings: (list.postings ?? []).map((p) => ({
|
|
314
|
-
docId: p.doc_id,
|
|
315
|
-
termFrequency: p.term_freq,
|
|
316
|
-
field: p.field,
|
|
317
|
-
positions: p.positions ?? []
|
|
318
|
-
}))
|
|
319
|
-
};
|
|
320
|
-
}
|
|
321
|
-
const numeric = {};
|
|
322
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
323
|
-
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
324
|
-
}
|
|
325
|
-
const boolean = {};
|
|
326
|
-
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
327
|
-
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
328
|
-
}
|
|
329
|
-
const geopoint = {};
|
|
330
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
331
|
-
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
332
|
-
}
|
|
333
|
-
const vectorData = {};
|
|
334
|
-
for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
|
|
335
|
-
vectorData[field] = {
|
|
336
|
-
dimension: data.dimension,
|
|
337
|
-
vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
338
|
-
hnswGraph: data.hnsw_graph ? {
|
|
339
|
-
entryPoint: data.hnsw_graph.entry_point,
|
|
340
|
-
maxLayer: data.hnsw_graph.max_layer,
|
|
341
|
-
m: data.hnsw_graph.m,
|
|
342
|
-
efConstruction: data.hnsw_graph.ef_construction,
|
|
343
|
-
metric: validateHnswMetric(data.hnsw_graph.metric),
|
|
344
|
-
nodes: data.hnsw_graph.nodes
|
|
345
|
-
} : null,
|
|
346
|
-
codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
|
|
347
|
-
};
|
|
348
|
-
}
|
|
349
|
-
return {
|
|
350
|
-
indexName: raw.index_name,
|
|
351
|
-
partitionId: raw.partition_id,
|
|
352
|
-
totalPartitions: raw.total_partitions,
|
|
353
|
-
language: raw.language ?? "english",
|
|
354
|
-
schema: raw.schema ?? {},
|
|
355
|
-
docCount: raw.doc_count ?? 0,
|
|
356
|
-
avgDocLength: raw.avg_doc_length ?? 0,
|
|
357
|
-
documents,
|
|
358
|
-
invertedIndex,
|
|
359
|
-
fieldIndexes: {
|
|
360
|
-
numeric,
|
|
361
|
-
boolean,
|
|
362
|
-
enum: raw.field_indexes?.enum ?? {},
|
|
363
|
-
geopoint,
|
|
364
|
-
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
365
|
-
},
|
|
366
|
-
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
367
|
-
vectorData,
|
|
368
|
-
statistics: {
|
|
369
|
-
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
370
|
-
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
371
|
-
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
372
|
-
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
373
|
-
}
|
|
374
|
-
};
|
|
375
|
-
}
|
|
376
|
-
function deserializePayloadV1(data) {
|
|
377
|
-
const raw = decode(data);
|
|
378
|
-
return wireToPartition(raw);
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
// src/serialization/payload-v1/sync-metadata.ts
|
|
382
|
-
function invalid(message, details) {
|
|
383
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The device sync state in the index metadata ${message}`, details);
|
|
384
|
-
}
|
|
385
|
-
function isPositiveSafeInteger(value) {
|
|
386
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
|
|
387
|
-
}
|
|
388
|
-
function syncMetadataToWire(meta) {
|
|
389
|
-
const wire = {};
|
|
390
|
-
if (meta.sync !== void 0) {
|
|
391
|
-
wire.sync = {
|
|
392
|
-
enabled: meta.sync.enabled,
|
|
393
|
-
...meta.sync.historyMaxAgeMs !== void 0 ? { history_max_age_ms: meta.sync.historyMaxAgeMs } : {},
|
|
394
|
-
...meta.sync.historyMaxBytes !== void 0 ? { history_max_bytes: meta.sync.historyMaxBytes } : {}
|
|
395
|
-
};
|
|
396
|
-
}
|
|
397
|
-
if (meta.syncActive !== void 0) wire.sync_active = meta.syncActive;
|
|
398
|
-
if (meta.syncEpoch !== void 0) wire.sync_epoch = meta.syncEpoch;
|
|
399
|
-
return wire;
|
|
400
|
-
}
|
|
401
|
-
function storedSyncSetting(raw) {
|
|
402
|
-
if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
|
|
403
|
-
const { enabled, history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
|
|
404
|
-
if (typeof enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
|
|
405
|
-
if (maxAge !== void 0 && !isPositiveSafeInteger(maxAge)) {
|
|
406
|
-
invalid("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
|
|
407
|
-
}
|
|
408
|
-
if (maxBytes !== void 0 && !isPositiveSafeInteger(maxBytes)) {
|
|
409
|
-
invalid("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
|
|
410
|
-
}
|
|
411
|
-
return {
|
|
412
|
-
enabled,
|
|
413
|
-
...maxAge !== void 0 ? { historyMaxAgeMs: maxAge } : {},
|
|
414
|
-
...maxBytes !== void 0 ? { historyMaxBytes: maxBytes } : {}
|
|
415
|
-
};
|
|
416
|
-
}
|
|
417
|
-
function syncMetadataFromWire(raw, meta) {
|
|
418
|
-
if (raw.sync !== void 0 && raw.sync !== null) meta.sync = storedSyncSetting(raw.sync);
|
|
419
|
-
if (raw.sync_active !== void 0 && raw.sync_active !== null) {
|
|
420
|
-
if (typeof raw.sync_active !== "boolean") invalid("holds a sync_active that is not a boolean", {});
|
|
421
|
-
meta.syncActive = raw.sync_active;
|
|
422
|
-
}
|
|
423
|
-
if (raw.sync_epoch !== void 0 && raw.sync_epoch !== null) {
|
|
424
|
-
if (!isSyncEpoch(raw.sync_epoch)) invalid("holds an epoch that is not 32 lowercase hexadecimal digits", {});
|
|
425
|
-
meta.syncEpoch = raw.sync_epoch;
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
// src/serialization/payload-v1/metadata.ts
|
|
430
|
-
function metadataToWire(meta) {
|
|
431
|
-
const wire = {
|
|
432
|
-
index_name: meta.indexName,
|
|
433
|
-
schema: meta.schema,
|
|
434
|
-
language: meta.language,
|
|
435
|
-
partition_count: meta.partitionCount,
|
|
436
|
-
bm25_params: meta.bm25Params,
|
|
437
|
-
created_at: meta.createdAt,
|
|
438
|
-
engine_version: meta.engineVersion
|
|
439
|
-
};
|
|
440
|
-
if (meta.vectorFields) {
|
|
441
|
-
wire.vector_fields = meta.vectorFields;
|
|
442
|
-
}
|
|
443
|
-
if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
|
|
444
|
-
wire.document_count = meta.documentCount;
|
|
445
|
-
}
|
|
446
|
-
if (meta.embedding) {
|
|
447
|
-
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
448
|
-
}
|
|
449
|
-
if (meta.surfaceForms !== void 0) {
|
|
450
|
-
wire.surface_forms_enabled = meta.surfaceForms;
|
|
451
|
-
}
|
|
452
|
-
if (meta.analysisRevision !== void 0) {
|
|
453
|
-
wire.analysis_revision = meta.analysisRevision;
|
|
454
|
-
}
|
|
455
|
-
if (meta.arrayPositionGap !== void 0) {
|
|
456
|
-
wire.array_position_gap = meta.arrayPositionGap;
|
|
457
|
-
}
|
|
458
|
-
if (meta.tokenizer !== void 0) {
|
|
459
|
-
wire.tokenizer = meta.tokenizer;
|
|
460
|
-
}
|
|
461
|
-
if (meta.stopWords !== void 0) {
|
|
462
|
-
wire.stop_words = meta.stopWords;
|
|
463
|
-
}
|
|
464
|
-
if (meta.stopWordList !== void 0) {
|
|
465
|
-
wire.stop_word_list = meta.stopWordList;
|
|
466
|
-
}
|
|
467
|
-
if (meta.analysisGeneration !== void 0) {
|
|
468
|
-
wire.analysis_generation = meta.analysisGeneration;
|
|
469
|
-
}
|
|
470
|
-
if (meta.pendingAnalysis !== void 0) {
|
|
471
|
-
const pending = meta.pendingAnalysis;
|
|
472
|
-
wire.pending_analysis = {
|
|
473
|
-
generation: pending.generation,
|
|
474
|
-
language: pending.language,
|
|
475
|
-
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
476
|
-
...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
|
|
477
|
-
...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
|
|
478
|
-
};
|
|
479
|
-
}
|
|
480
|
-
if (meta.partitionLimits !== void 0) {
|
|
481
|
-
wire.partition_limits = {
|
|
482
|
-
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
483
|
-
...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
|
|
484
|
-
...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
|
|
485
|
-
};
|
|
486
|
-
}
|
|
487
|
-
if (meta.defaultScoring !== void 0) {
|
|
488
|
-
wire.default_scoring = meta.defaultScoring;
|
|
489
|
-
}
|
|
490
|
-
if (meta.trackPositions !== void 0) {
|
|
491
|
-
wire.track_positions = meta.trackPositions;
|
|
492
|
-
}
|
|
493
|
-
if (meta.textStorage !== void 0) {
|
|
494
|
-
wire.text_storage = meta.textStorage;
|
|
495
|
-
}
|
|
496
|
-
if (meta.strict !== void 0) {
|
|
497
|
-
wire.strict = meta.strict;
|
|
498
|
-
}
|
|
499
|
-
if (meta.required !== void 0) {
|
|
500
|
-
wire.required = meta.required;
|
|
501
|
-
}
|
|
502
|
-
if (meta.vectorPromotion !== void 0) {
|
|
503
|
-
const promotion = meta.vectorPromotion;
|
|
504
|
-
wire.vector_promotion = {
|
|
505
|
-
...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
|
|
506
|
-
...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
|
|
507
|
-
...promotion.hnswConfig !== void 0 ? {
|
|
508
|
-
hnsw_config: {
|
|
509
|
-
...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
|
|
510
|
-
...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
|
|
511
|
-
...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
|
|
512
|
-
}
|
|
513
|
-
} : {},
|
|
514
|
-
...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
|
|
515
|
-
...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
|
|
516
|
-
};
|
|
517
|
-
}
|
|
518
|
-
if (meta.patternValueLimit !== void 0) {
|
|
519
|
-
wire.pattern_value_limit = meta.patternValueLimit;
|
|
520
|
-
}
|
|
521
|
-
if (meta.indexUuid !== void 0) {
|
|
522
|
-
wire.index_uuid = meta.indexUuid;
|
|
523
|
-
}
|
|
524
|
-
if (meta.heldPartitions !== void 0) {
|
|
525
|
-
wire.held_partitions = meta.heldPartitions;
|
|
526
|
-
}
|
|
527
|
-
if (Array.isArray(meta.synonyms) && meta.synonyms.length > 0) {
|
|
528
|
-
wire.synonyms = meta.synonyms;
|
|
529
|
-
}
|
|
530
|
-
return {
|
|
531
|
-
...wire,
|
|
532
|
-
...syncMetadataToWire(meta),
|
|
533
|
-
...meta.querySuggestions !== void 0 ? storedStateToWire(meta.querySuggestions) : {}
|
|
534
|
-
};
|
|
535
|
-
}
|
|
536
|
-
function wireToMetadata(raw) {
|
|
537
|
-
const meta = {
|
|
538
|
-
indexName: raw.index_name,
|
|
539
|
-
schema: raw.schema ?? {},
|
|
540
|
-
language: raw.language ?? "english",
|
|
541
|
-
partitionCount: raw.partition_count ?? 1,
|
|
542
|
-
bm25Params: raw.bm25_params ?? { k1: DEFAULT_BM25_K1, b: DEFAULT_BM25_B },
|
|
543
|
-
createdAt: raw.created_at ?? 0,
|
|
544
|
-
engineVersion: raw.engine_version ?? "0.0.0"
|
|
545
|
-
};
|
|
546
|
-
if (raw.vector_fields) {
|
|
547
|
-
for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
|
|
548
|
-
if (!isQuantizationMode(field.quantization)) {
|
|
549
|
-
throw new NarsilError(
|
|
550
|
-
ErrorCodes.CONFIG_INVALID,
|
|
551
|
-
`Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
|
|
552
|
-
{ fieldPath, quantization: field.quantization }
|
|
553
|
-
);
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
meta.vectorFields = raw.vector_fields;
|
|
557
|
-
}
|
|
558
|
-
if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
|
|
559
|
-
meta.documentCount = raw.document_count;
|
|
560
|
-
}
|
|
561
|
-
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
562
|
-
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
563
|
-
}
|
|
564
|
-
if (typeof raw.surface_forms_enabled === "boolean") {
|
|
565
|
-
meta.surfaceForms = raw.surface_forms_enabled;
|
|
566
|
-
}
|
|
567
|
-
if (typeof raw.analysis_revision === "string") {
|
|
568
|
-
meta.analysisRevision = raw.analysis_revision;
|
|
569
|
-
}
|
|
570
|
-
if (typeof raw.array_position_gap === "number") {
|
|
571
|
-
meta.arrayPositionGap = raw.array_position_gap;
|
|
572
|
-
}
|
|
573
|
-
if (typeof raw.tokenizer === "string") {
|
|
574
|
-
meta.tokenizer = raw.tokenizer;
|
|
575
|
-
}
|
|
576
|
-
if (typeof raw.stop_words === "string") {
|
|
577
|
-
meta.stopWords = raw.stop_words;
|
|
578
|
-
}
|
|
579
|
-
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
580
|
-
meta.stopWordList = raw.stop_word_list;
|
|
581
|
-
}
|
|
582
|
-
if (isPositiveInteger(raw.analysis_generation)) {
|
|
583
|
-
meta.analysisGeneration = raw.analysis_generation;
|
|
584
|
-
}
|
|
585
|
-
const pending = raw.pending_analysis;
|
|
586
|
-
if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
|
|
587
|
-
meta.pendingAnalysis = {
|
|
588
|
-
generation: pending.generation,
|
|
589
|
-
language: pending.language,
|
|
590
|
-
...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
|
|
591
|
-
...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
|
|
592
|
-
...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
|
|
593
|
-
};
|
|
594
|
-
}
|
|
595
|
-
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
596
|
-
const limits = {
|
|
597
|
-
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
598
|
-
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
|
|
599
|
-
...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
|
|
600
|
-
};
|
|
601
|
-
if (Object.keys(limits).length > 0) {
|
|
602
|
-
meta.partitionLimits = limits;
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
|
|
606
|
-
meta.defaultScoring = raw.default_scoring;
|
|
607
|
-
}
|
|
608
|
-
if (typeof raw.track_positions === "boolean") {
|
|
609
|
-
meta.trackPositions = raw.track_positions;
|
|
610
|
-
}
|
|
611
|
-
if (raw.text_storage !== void 0) {
|
|
612
|
-
if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
|
|
613
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
|
|
614
|
-
textStorage: raw.text_storage
|
|
615
|
-
});
|
|
616
|
-
}
|
|
617
|
-
meta.textStorage = raw.text_storage;
|
|
618
|
-
}
|
|
619
|
-
if (typeof raw.strict === "boolean") {
|
|
620
|
-
meta.strict = raw.strict;
|
|
621
|
-
}
|
|
622
|
-
if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
|
|
623
|
-
meta.required = raw.required;
|
|
624
|
-
}
|
|
625
|
-
if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
|
|
626
|
-
const promotion = raw.vector_promotion;
|
|
627
|
-
const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
|
|
628
|
-
const metric = hnsw?.metric;
|
|
629
|
-
const restored = {};
|
|
630
|
-
const hnswConfig = {};
|
|
631
|
-
if (isPositiveInteger(hnsw?.m)) {
|
|
632
|
-
hnswConfig.m = hnsw?.m;
|
|
633
|
-
}
|
|
634
|
-
if (isPositiveInteger(hnsw?.ef_construction)) {
|
|
635
|
-
hnswConfig.efConstruction = hnsw?.ef_construction;
|
|
636
|
-
}
|
|
637
|
-
if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
|
|
638
|
-
hnswConfig.metric = metric;
|
|
639
|
-
}
|
|
640
|
-
if (isPositiveInteger(promotion.threshold)) {
|
|
641
|
-
restored.threshold = promotion.threshold;
|
|
642
|
-
}
|
|
643
|
-
if (isSelectivityRatio(promotion.filter_threshold)) {
|
|
644
|
-
restored.filterThreshold = promotion.filter_threshold;
|
|
645
|
-
}
|
|
646
|
-
if (Object.keys(hnswConfig).length > 0) {
|
|
647
|
-
restored.hnswConfig = hnswConfig;
|
|
648
|
-
}
|
|
649
|
-
if (promotion.quantization !== void 0) {
|
|
650
|
-
if (!isQuantizationMode(promotion.quantization)) {
|
|
651
|
-
throw new NarsilError(
|
|
652
|
-
ErrorCodes.CONFIG_INVALID,
|
|
653
|
-
`Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
|
|
654
|
-
{ quantization: promotion.quantization }
|
|
655
|
-
);
|
|
656
|
-
}
|
|
657
|
-
restored.quantization = promotion.quantization;
|
|
658
|
-
}
|
|
659
|
-
if (promotion.storage !== void 0) {
|
|
660
|
-
if (!isStorageMode(promotion.storage)) {
|
|
661
|
-
throw new NarsilError(
|
|
662
|
-
ErrorCodes.CONFIG_INVALID,
|
|
663
|
-
`Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
|
|
664
|
-
{ storage: promotion.storage }
|
|
665
|
-
);
|
|
666
|
-
}
|
|
667
|
-
restored.storage = promotion.storage;
|
|
668
|
-
}
|
|
669
|
-
if (Object.keys(restored).length > 0) {
|
|
670
|
-
meta.vectorPromotion = restored;
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
if (raw.pattern_value_limit !== void 0) {
|
|
674
|
-
if (!isPositiveInteger(raw.pattern_value_limit)) {
|
|
675
|
-
throw new NarsilError(
|
|
676
|
-
ErrorCodes.CONFIG_INVALID,
|
|
677
|
-
`The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
|
|
678
|
-
{ patternValueLimit: String(raw.pattern_value_limit) }
|
|
679
|
-
);
|
|
680
|
-
}
|
|
681
|
-
meta.patternValueLimit = raw.pattern_value_limit;
|
|
682
|
-
}
|
|
683
|
-
if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
|
|
684
|
-
meta.indexUuid = raw.index_uuid;
|
|
685
|
-
}
|
|
686
|
-
if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
|
|
687
|
-
meta.heldPartitions = raw.held_partitions;
|
|
688
|
-
}
|
|
689
|
-
if (raw.synonyms !== void 0 && raw.synonyms !== null) meta.synonyms = raw.synonyms;
|
|
690
|
-
syncMetadataFromWire(raw, meta);
|
|
691
|
-
const querySuggestions = storedStateFromWire(raw);
|
|
692
|
-
if (querySuggestions !== null) meta.querySuggestions = querySuggestions;
|
|
693
|
-
return meta;
|
|
694
|
-
}
|
|
695
|
-
function isNonNegativeInteger(value) {
|
|
696
|
-
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
697
|
-
}
|
|
698
|
-
function isStringList(value) {
|
|
699
|
-
return Array.isArray(value) && value.every((word) => typeof word === "string");
|
|
700
|
-
}
|
|
701
|
-
function isPositiveInteger(value) {
|
|
702
|
-
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
703
|
-
}
|
|
704
|
-
function isSelectivityRatio(value) {
|
|
705
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
706
|
-
}
|
|
707
|
-
function isWatermarkFraction(value) {
|
|
708
|
-
return typeof value === "number" && value > 0 && value <= 1;
|
|
709
|
-
}
|
|
710
|
-
function serializeMetadata(meta) {
|
|
711
|
-
const wire = metadataToWire(meta);
|
|
712
|
-
return encode(wire);
|
|
713
|
-
}
|
|
714
|
-
function deserializeMetadata(data) {
|
|
715
|
-
const raw = decode(data);
|
|
716
|
-
return wireToMetadata(raw);
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
// src/serialization/envelope.ts
|
|
720
|
-
var CURRENT_ENVELOPE_VERSION = 1;
|
|
721
|
-
var SNAPSHOT_ENVELOPE_VERSION = 2;
|
|
722
|
-
var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
|
|
723
|
-
function buildFlags(options) {
|
|
724
|
-
const useCompression = options.compression === "gzip";
|
|
725
|
-
return {
|
|
726
|
-
compressionEnabled: useCompression,
|
|
727
|
-
compressionAlgorithm: useCompression ? "gzip" : "none",
|
|
728
|
-
checksumPresent: options.checksum ?? false,
|
|
729
|
-
encryptionEnabled: false
|
|
730
|
-
};
|
|
731
|
-
}
|
|
732
|
-
async function packEnvelope(payloadBytes, options) {
|
|
733
|
-
const flags = buildFlags(options);
|
|
734
|
-
let finalPayload = payloadBytes;
|
|
735
|
-
if (flags.compressionEnabled) {
|
|
736
|
-
finalPayload = await gzip(payloadBytes);
|
|
737
|
-
}
|
|
738
|
-
const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
|
|
739
|
-
const [major, minor, patch] = engineVersionParts();
|
|
740
|
-
const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
|
|
741
|
-
const header = {
|
|
742
|
-
magic: "NRSL",
|
|
743
|
-
envelopeFormatVersion,
|
|
744
|
-
engineVersionMajor: major,
|
|
745
|
-
engineVersionMinor: minor,
|
|
746
|
-
engineVersionPatch: patch,
|
|
747
|
-
payloadLength: finalPayload.length,
|
|
748
|
-
flags,
|
|
749
|
-
checksum,
|
|
750
|
-
reserved: new Uint8Array(14)
|
|
751
|
-
};
|
|
752
|
-
const headerBytes = writeHeader(header);
|
|
753
|
-
const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
|
|
754
|
-
envelope.set(headerBytes, 0);
|
|
755
|
-
envelope.set(finalPayload, HEADER_SIZE);
|
|
756
|
-
return envelope;
|
|
757
|
-
}
|
|
758
|
-
async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
|
|
759
|
-
const header = readHeader(data);
|
|
760
|
-
if (header.envelopeFormatVersion > maxAcceptedVersion) {
|
|
761
|
-
throw new NarsilError(
|
|
762
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
763
|
-
`This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
|
|
764
|
-
);
|
|
765
|
-
}
|
|
766
|
-
if (header.flags.encryptionEnabled) {
|
|
767
|
-
throw new NarsilError(
|
|
768
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
769
|
-
"Encrypted envelopes are not supported in this version of Narsil"
|
|
770
|
-
);
|
|
771
|
-
}
|
|
772
|
-
const compAlgo = header.flags.compressionAlgorithm;
|
|
773
|
-
if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
|
|
774
|
-
throw new NarsilError(
|
|
775
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
776
|
-
`Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
|
|
777
|
-
);
|
|
778
|
-
}
|
|
779
|
-
const availablePayload = data.length - HEADER_SIZE;
|
|
780
|
-
if (availablePayload < header.payloadLength) {
|
|
781
|
-
throw new NarsilError(
|
|
782
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
783
|
-
`Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
|
|
784
|
-
);
|
|
785
|
-
}
|
|
786
|
-
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
|
|
787
|
-
if (header.flags.checksumPresent) {
|
|
788
|
-
const computed = crc32(payloadBytes);
|
|
789
|
-
if (computed !== header.checksum) {
|
|
790
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
|
|
791
|
-
expected: header.checksum,
|
|
792
|
-
computed
|
|
793
|
-
});
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
if (header.flags.compressionEnabled) {
|
|
797
|
-
payloadBytes = await gunzip(payloadBytes);
|
|
798
|
-
}
|
|
799
|
-
return { header, payloadBytes };
|
|
800
|
-
}
|
|
801
|
-
async function unpackEnvelopeBytes(data) {
|
|
802
|
-
return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
|
|
803
|
-
}
|
|
804
|
-
async function unpackIndexSnapshotEnvelope(data) {
|
|
805
|
-
const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
|
|
806
|
-
if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
|
|
807
|
-
throw new NarsilError(
|
|
808
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
809
|
-
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
|
|
810
|
-
);
|
|
811
|
-
}
|
|
812
|
-
return payloadBytes;
|
|
813
|
-
}
|
|
814
|
-
async function writeMetadataEnvelope(meta, options = {}) {
|
|
815
|
-
const payloadBytes = serializeMetadata(meta);
|
|
816
|
-
return packEnvelope(payloadBytes, options);
|
|
817
|
-
}
|
|
818
|
-
async function readMetadataEnvelope(data) {
|
|
819
|
-
const { header, payloadBytes } = await unpackEnvelope(data);
|
|
820
|
-
const metadata = deserializeMetadata(payloadBytes);
|
|
821
|
-
return { header, metadata };
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
// src/serialization/crc32-combine.ts
|
|
825
|
-
var IEEE_POLYNOMIAL = 3988292384;
|
|
826
|
-
var CRC_BITS = 32;
|
|
827
|
-
var zeroByteOperators = [];
|
|
828
|
-
function timesVector(operator, vector) {
|
|
829
|
-
let sum = 0;
|
|
830
|
-
let rest = vector >>> 0;
|
|
831
|
-
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
832
|
-
if (rest & 1) sum ^= operator[row];
|
|
833
|
-
}
|
|
834
|
-
return sum >>> 0;
|
|
835
|
-
}
|
|
836
|
-
function squared(operator) {
|
|
837
|
-
const result = new Uint32Array(CRC_BITS);
|
|
838
|
-
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
839
|
-
return result;
|
|
840
|
-
}
|
|
841
|
-
function firstOperator() {
|
|
842
|
-
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
843
|
-
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
844
|
-
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
845
|
-
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
846
|
-
return oneZeroBit;
|
|
847
|
-
}
|
|
848
|
-
function operatorForZeroBytesPower(power) {
|
|
849
|
-
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
850
|
-
while (zeroByteOperators.length <= power) {
|
|
851
|
-
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
852
|
-
}
|
|
853
|
-
return zeroByteOperators[power];
|
|
854
|
-
}
|
|
855
|
-
function crc32Combine(first, second, secondByteLength) {
|
|
856
|
-
let crc = first >>> 0;
|
|
857
|
-
let rest = secondByteLength;
|
|
858
|
-
for (let power = 0; rest > 0; power++) {
|
|
859
|
-
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
860
|
-
rest = Math.floor(rest / 2);
|
|
861
|
-
}
|
|
862
|
-
return (crc ^ second) >>> 0;
|
|
863
|
-
}
|
|
864
|
-
var BIN8 = 196;
|
|
865
|
-
var BIN16 = 197;
|
|
866
|
-
var BIN32 = 198;
|
|
867
|
-
var ARRAY16 = 220;
|
|
868
|
-
var ARRAY32 = 221;
|
|
869
|
-
var FIXMAP = 128;
|
|
870
|
-
var FIXARRAY = 144;
|
|
871
|
-
var FIXED_LIST_MAX = 15;
|
|
872
|
-
var FIXSTR = 160;
|
|
873
|
-
var FIXSTR_MAX_LENGTH = 31;
|
|
874
|
-
var ASCII_MAX = 127;
|
|
875
|
-
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
876
|
-
var STRING_HEADER_MAX_BYTES = 5;
|
|
877
|
-
var BIN8_MAX_BYTES = 255;
|
|
878
|
-
var UINT16_MAX = 65535;
|
|
879
|
-
function binHeader(byteLength) {
|
|
880
|
-
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
881
|
-
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
882
|
-
const header = new Uint8Array(5);
|
|
883
|
-
header[0] = BIN32;
|
|
884
|
-
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
885
|
-
return header;
|
|
886
|
-
}
|
|
887
|
-
function arrayHeader(count) {
|
|
888
|
-
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
889
|
-
if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
|
|
890
|
-
const header = new Uint8Array(5);
|
|
891
|
-
header[0] = ARRAY32;
|
|
892
|
-
new DataView(header.buffer).setUint32(1, count, false);
|
|
893
|
-
return header;
|
|
894
|
-
}
|
|
895
|
-
function createPayloadBuilder() {
|
|
896
|
-
const chunks = [];
|
|
897
|
-
let offset = 0;
|
|
898
|
-
function push(chunk) {
|
|
899
|
-
chunks.push(chunk);
|
|
900
|
-
offset += chunk.byteLength;
|
|
901
|
-
}
|
|
902
|
-
return {
|
|
903
|
-
chunks,
|
|
904
|
-
get offset() {
|
|
905
|
-
return offset;
|
|
906
|
-
},
|
|
907
|
-
push,
|
|
908
|
-
entry(key, value) {
|
|
909
|
-
push(encode(key));
|
|
910
|
-
push(encode(value));
|
|
911
|
-
},
|
|
912
|
-
rawEntry(key, encoded) {
|
|
913
|
-
push(encode(key));
|
|
914
|
-
const at = offset;
|
|
915
|
-
push(encoded);
|
|
916
|
-
return at;
|
|
917
|
-
},
|
|
918
|
-
bytesEntry(key, bytes) {
|
|
919
|
-
push(encode(key));
|
|
920
|
-
push(binHeader(bytes.byteLength));
|
|
921
|
-
const at = offset;
|
|
922
|
-
push(bytes);
|
|
923
|
-
return at;
|
|
924
|
-
}
|
|
925
|
-
};
|
|
926
|
-
}
|
|
927
|
-
function pageChecksums(bytes, pageBytes) {
|
|
928
|
-
const pages = Math.ceil(bytes.byteLength / pageBytes);
|
|
929
|
-
const checksums = new Uint8Array(pages * 4);
|
|
930
|
-
const view = new DataView(checksums.buffer);
|
|
931
|
-
for (let page = 0; page < pages; page++) {
|
|
932
|
-
const from = page * pageBytes;
|
|
933
|
-
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
|
|
934
|
-
}
|
|
935
|
-
return checksums;
|
|
936
|
-
}
|
|
937
|
-
function sectionOf(name, offset, bytes, pageBytes) {
|
|
938
|
-
return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
|
|
939
|
-
}
|
|
940
|
-
function uint32Bytes(values) {
|
|
941
|
-
const bytes = new Uint8Array(values.length * 4);
|
|
942
|
-
const view = new DataView(bytes.buffer);
|
|
943
|
-
for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
|
|
944
|
-
return bytes;
|
|
945
|
-
}
|
|
946
|
-
function writtenAsShortAscii(docId, list, at) {
|
|
947
|
-
if (docId.length > FIXSTR_MAX_LENGTH) return false;
|
|
948
|
-
for (let unit = 0; unit < docId.length; unit++) {
|
|
949
|
-
const code = docId.charCodeAt(unit);
|
|
950
|
-
if (code > ASCII_MAX) return false;
|
|
951
|
-
list[at + 1 + unit] = code;
|
|
952
|
-
}
|
|
953
|
-
list[at] = FIXSTR | docId.length;
|
|
954
|
-
return true;
|
|
955
|
-
}
|
|
956
|
-
function encodedIdList(docIds) {
|
|
957
|
-
const header = arrayHeader(docIds.length);
|
|
958
|
-
let capacity = header.byteLength;
|
|
959
|
-
for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
|
|
960
|
-
const list = new Uint8Array(capacity);
|
|
961
|
-
list.set(header);
|
|
962
|
-
const offsets = new Uint32Array(docIds.length + 1);
|
|
963
|
-
let cursor = header.byteLength;
|
|
964
|
-
for (let i = 0; i < docIds.length; i++) {
|
|
965
|
-
offsets[i] = cursor;
|
|
966
|
-
if (writtenAsShortAscii(docIds[i], list, cursor)) {
|
|
967
|
-
cursor += docIds[i].length + 1;
|
|
968
|
-
continue;
|
|
969
|
-
}
|
|
970
|
-
const encoded = encode(docIds[i]);
|
|
971
|
-
list.set(encoded, cursor);
|
|
972
|
-
cursor += encoded.byteLength;
|
|
973
|
-
}
|
|
974
|
-
offsets[docIds.length] = cursor;
|
|
975
|
-
return { list: list.subarray(0, cursor), offsets };
|
|
976
|
-
}
|
|
977
|
-
function positionsInIdOrder(docIds) {
|
|
978
|
-
const positions = Array.from(docIds.keys());
|
|
979
|
-
positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
|
|
980
|
-
return Uint32Array.from(positions);
|
|
981
|
-
}
|
|
982
|
-
function magnitudeBytes(magnitudes) {
|
|
983
|
-
const bytes = new Uint8Array(magnitudes.length * 8);
|
|
984
|
-
const view = new DataView(bytes.buffer);
|
|
985
|
-
for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
|
|
986
|
-
return bytes;
|
|
987
|
-
}
|
|
988
|
-
function checksumAcrossPages(checksums, byteLength, pageBytes) {
|
|
989
|
-
const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
|
|
990
|
-
let whole = 0;
|
|
991
|
-
for (let page = 0; page * 4 < checksums.byteLength; page++) {
|
|
992
|
-
const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
|
|
993
|
-
whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
|
|
994
|
-
}
|
|
995
|
-
return whole;
|
|
996
|
-
}
|
|
997
|
-
function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
|
|
998
|
-
let headLength = 0;
|
|
999
|
-
let head = new Uint8Array(0);
|
|
1000
|
-
let vectorsSection = {
|
|
1001
|
-
name: "vectors",
|
|
1002
|
-
offset: 0,
|
|
1003
|
-
length: vectors.byteLength,
|
|
1004
|
-
pageBytes: SEGMENT_PAGE_BYTES,
|
|
1005
|
-
checksums
|
|
1006
|
-
};
|
|
1007
|
-
for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
|
|
1008
|
-
vectorsSection = {
|
|
1009
|
-
...vectorsSection,
|
|
1010
|
-
offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
|
|
1011
|
-
};
|
|
1012
|
-
head = encode({ sections: [...sections, vectorsSection] });
|
|
1013
|
-
if (head.byteLength === headLength) break;
|
|
1014
|
-
headLength = head.byteLength;
|
|
1015
|
-
}
|
|
1016
|
-
return { head, vectorsSection };
|
|
1017
|
-
}
|
|
1018
|
-
function pagedVectorFileChunks(payload, heldMagnitudes) {
|
|
1019
|
-
const builder = createPayloadBuilder();
|
|
1020
|
-
const { codes } = payload;
|
|
1021
|
-
builder.push(Uint8Array.of(FIXMAP | 10));
|
|
1022
|
-
builder.entry("v", payload.v);
|
|
1023
|
-
builder.entry("fieldName", payload.fieldName);
|
|
1024
|
-
builder.entry("dimension", payload.dimension);
|
|
1025
|
-
const { list, offsets } = encodedIdList(payload.docIds);
|
|
1026
|
-
const docIdsAt = builder.rawEntry("docIds", list);
|
|
1027
|
-
const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
|
|
1028
|
-
builder.push(encode("codes"));
|
|
1029
|
-
if (codes === null) {
|
|
1030
|
-
builder.push(encode(null));
|
|
1031
|
-
} else {
|
|
1032
|
-
builder.push(Uint8Array.of(FIXMAP | 3));
|
|
1033
|
-
builder.entry("bits", codes.bits);
|
|
1034
|
-
builder.entry("centroid", codes.centroid);
|
|
1035
|
-
const recordsAt = builder.bytesEntry("records", codes.records);
|
|
1036
|
-
sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
|
|
1037
|
-
}
|
|
1038
|
-
const magnitudes = magnitudeBytes(heldMagnitudes);
|
|
1039
|
-
sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
|
|
1040
|
-
const idOffsets = uint32Bytes(offsets);
|
|
1041
|
-
sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
|
|
1042
|
-
const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
|
|
1043
|
-
sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
|
|
1044
|
-
const headKey = encode("head");
|
|
1045
|
-
const vectorsKey = encode("vectors");
|
|
1046
|
-
const vectorsHeader = binHeader(payload.vectors.byteLength);
|
|
1047
|
-
const beforeHead = builder.offset + headKey.byteLength;
|
|
1048
|
-
const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
|
|
1049
|
-
const { head } = encodedHeadAndVectorsPlace(
|
|
1050
|
-
sections,
|
|
1051
|
-
(headLength) => beforeHead + binHeader(headLength).byteLength,
|
|
1052
|
-
payload.vectors,
|
|
1053
|
-
vectorChecksums,
|
|
1054
|
-
vectorsKey.byteLength + vectorsHeader.byteLength
|
|
1055
|
-
);
|
|
1056
|
-
const headOffset = builder.bytesEntry("head", head);
|
|
1057
|
-
builder.push(vectorsKey);
|
|
1058
|
-
builder.push(vectorsHeader);
|
|
1059
|
-
const beforeVectors = [...builder.chunks];
|
|
1060
|
-
builder.push(payload.vectors);
|
|
1061
|
-
return {
|
|
1062
|
-
chunks: builder.chunks,
|
|
1063
|
-
headOffset,
|
|
1064
|
-
headLength: head.byteLength,
|
|
1065
|
-
headChecksum: crc32(head),
|
|
1066
|
-
beforeVectors,
|
|
1067
|
-
vectors: {
|
|
1068
|
-
byteLength: payload.vectors.byteLength,
|
|
1069
|
-
checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
|
|
1070
|
-
}
|
|
1071
|
-
};
|
|
1072
|
-
}
|
|
1073
|
-
function nodeOffsetsOf(graph) {
|
|
1074
|
-
const offsets = new Uint32Array(graph.levels.length + 1);
|
|
1075
|
-
const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
|
|
1076
|
-
let cursor = 0;
|
|
1077
|
-
for (let at = 0; at < graph.levels.length; at++) {
|
|
1078
|
-
offsets[at] = cursor;
|
|
1079
|
-
for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
|
|
1080
|
-
}
|
|
1081
|
-
offsets[graph.levels.length] = cursor;
|
|
1082
|
-
return uint32Bytes(offsets);
|
|
1083
|
-
}
|
|
1084
|
-
function pagedGraphPartChunks(part) {
|
|
1085
|
-
const builder = createPayloadBuilder();
|
|
1086
|
-
builder.push(Uint8Array.of(FIXMAP | 4));
|
|
1087
|
-
builder.entry("v", part.v);
|
|
1088
|
-
builder.entry("fieldName", part.fieldName);
|
|
1089
|
-
builder.push(encode("graphs"));
|
|
1090
|
-
builder.push(arrayHeader(part.graphs.length));
|
|
1091
|
-
const layouts = [];
|
|
1092
|
-
for (const graph of part.graphs) {
|
|
1093
|
-
const { levels, neighbours, ...header } = graph;
|
|
1094
|
-
const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
|
|
1095
|
-
builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
|
|
1096
|
-
for (const [key, value] of headerEntries) builder.entry(key, value);
|
|
1097
|
-
const levelsAt = builder.bytesEntry("levels", levels);
|
|
1098
|
-
const neighboursAt = builder.bytesEntry("neighbours", neighbours);
|
|
1099
|
-
layouts.push({
|
|
1100
|
-
...header,
|
|
1101
|
-
levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
|
|
1102
|
-
nodeOffsets: nodeOffsetsOf(graph),
|
|
1103
|
-
neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
|
|
1104
|
-
});
|
|
1105
|
-
}
|
|
1106
|
-
const head = encode({ graphs: layouts });
|
|
1107
|
-
const headOffset = builder.bytesEntry("head", head);
|
|
1108
|
-
return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
|
|
1109
|
-
}
|
|
1110
|
-
var EMPTY_BIN = new Uint8Array(0);
|
|
1111
|
-
var EMPTY_BIN_BYTES = 2;
|
|
1112
|
-
var BIN82 = 196;
|
|
1113
|
-
var BIN162 = 197;
|
|
1114
|
-
var BIN322 = 198;
|
|
1115
|
-
var BIN8_MAX_BYTES2 = 255;
|
|
1116
|
-
var BIN16_MAX_BYTES = 65535;
|
|
1117
|
-
function binHeader2(byteLength) {
|
|
1118
|
-
if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
|
|
1119
|
-
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1120
|
-
const header = new Uint8Array(5);
|
|
1121
|
-
header[0] = BIN322;
|
|
1122
|
-
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1123
|
-
return header;
|
|
1124
|
-
}
|
|
1125
|
-
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
1126
|
-
const head = encode(withEmptyBytesInTheirPlace);
|
|
1127
|
-
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
|
|
1128
|
-
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
1129
|
-
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
|
|
1130
|
-
}
|
|
1131
|
-
function vectorPartChunks(part) {
|
|
1132
|
-
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
1133
|
-
}
|
|
1134
|
-
|
|
1135
|
-
// src/persistence/durability/segment/vector-write.ts
|
|
1136
|
-
function keptFileRef(input, prior, kept) {
|
|
1137
|
-
const listed = prior?.files.find((file) => file.key === kept.key) ?? input.priorFileRefs?.get(kept.key);
|
|
1138
|
-
if (listed === void 0) {
|
|
1139
|
-
throw new NarsilError(
|
|
1140
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1141
|
-
`The checkpoint keeps the vector file "${kept.key}", which the previous manifest does not list`,
|
|
1142
|
-
{ key: kept.key }
|
|
1143
|
-
);
|
|
1144
|
-
}
|
|
1145
|
-
return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
|
|
1146
|
-
}
|
|
1147
|
-
function headFieldsOf(ref) {
|
|
1148
|
-
return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
|
|
1149
|
-
}
|
|
1150
|
-
function headRefOf(paged) {
|
|
1151
|
-
return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
|
|
1152
|
-
}
|
|
1153
|
-
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
1154
|
-
const parts = plan.graphParts ?? [];
|
|
1155
|
-
const refs = [];
|
|
1156
|
-
for (let part = 0; part < parts.length; part++) {
|
|
1157
|
-
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
1158
|
-
const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
|
|
1159
|
-
const packed = await packChecksummedEnvelope(
|
|
1160
|
-
paged === null ? [encode(parts[part])] : paged.chunks,
|
|
1161
|
-
GRAPH_PART_ENVELOPE_VERSION,
|
|
1162
|
-
`graph part "${key}"`
|
|
1163
|
-
);
|
|
1164
|
-
await input.store.write(key, packed.chunks);
|
|
1165
|
-
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
|
|
1166
|
-
}
|
|
1167
|
-
return refs;
|
|
1168
|
-
}
|
|
1169
|
-
async function writeField(input, fieldPath, plan, outcome) {
|
|
1170
|
-
const prior = input.priorVectors.find((ref) => ref.fieldPath === fieldPath);
|
|
1171
|
-
if (plan.unchanged && prior !== void 0) {
|
|
1172
|
-
outcome.refs.push(prior);
|
|
1173
|
-
return;
|
|
1174
|
-
}
|
|
1175
|
-
let nextFileId = prior?.nextFileId ?? 0;
|
|
1176
|
-
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
1177
|
-
const written = [];
|
|
1178
|
-
for (let index = 0; index < plan.newFiles; index++) {
|
|
1179
|
-
const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
|
|
1180
|
-
const id = nextFileId;
|
|
1181
|
-
nextFileId += 1;
|
|
1182
|
-
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
1183
|
-
const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
|
|
1184
|
-
const what = `vector file "${key}"`;
|
|
1185
|
-
const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
|
|
1186
|
-
paged.chunks,
|
|
1187
|
-
VECTOR_FILE_ENVELOPE_VERSION,
|
|
1188
|
-
what,
|
|
1189
|
-
crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
|
|
1190
|
-
);
|
|
1191
|
-
await input.store.write(key, packed.chunks);
|
|
1192
|
-
files.push({
|
|
1193
|
-
id,
|
|
1194
|
-
key,
|
|
1195
|
-
count: payload.docIds.length,
|
|
1196
|
-
dead,
|
|
1197
|
-
bytes: packed.byteLength,
|
|
1198
|
-
checksum: packed.checksum,
|
|
1199
|
-
...headRefOf(paged)
|
|
1200
|
-
});
|
|
1201
|
-
written.push({ id, key, ordinals });
|
|
1202
|
-
outcome.layouts.push({
|
|
1203
|
-
fieldPath,
|
|
1204
|
-
key,
|
|
1205
|
-
docIds: payload.docIds,
|
|
1206
|
-
dead,
|
|
1207
|
-
vectorsOffset: packed.byteLength - payload.vectors.byteLength
|
|
1208
|
-
});
|
|
1209
|
-
}
|
|
1210
|
-
const priorGeneration = prior?.graphGeneration ?? 0;
|
|
1211
|
-
const holdsGraph = plan.graphParts !== null && plan.graphParts.length > 0;
|
|
1212
|
-
const graphGeneration = holdsGraph ? priorGeneration + 1 : priorGeneration;
|
|
1213
|
-
const graphParts = holdsGraph ? await writeGraphParts(input, fieldPath, plan, graphGeneration) : [];
|
|
1214
|
-
outcome.refs.push({ fieldPath, nextFileId, files, graphGeneration, graphParts });
|
|
1215
|
-
outcome.written.push({ fieldPath, plan, files: written });
|
|
1216
|
-
}
|
|
1217
|
-
async function writeLiveVectors(input) {
|
|
1218
|
-
const outcome = { refs: [], layouts: [], written: [] };
|
|
1219
|
-
for (const [fieldPath, plan] of input.plans) await writeField(input, fieldPath, plan, outcome);
|
|
1220
|
-
return outcome;
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
// src/persistence/durability/seq-owner.ts
|
|
1224
|
-
var SINGLE_NODE_PRIMARY_TERM = 1;
|
|
1225
|
-
function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
|
|
1226
|
-
let current = startSeqNo;
|
|
1227
|
-
let term = primaryTerm;
|
|
1228
|
-
return {
|
|
1229
|
-
next() {
|
|
1230
|
-
current += 1;
|
|
1231
|
-
return current;
|
|
1232
|
-
},
|
|
1233
|
-
advanceTo(seqNo, nextPrimaryTerm) {
|
|
1234
|
-
current = seqNo;
|
|
1235
|
-
term = nextPrimaryTerm;
|
|
1236
|
-
},
|
|
1237
|
-
get current() {
|
|
1238
|
-
return current;
|
|
1239
|
-
},
|
|
1240
|
-
get primaryTerm() {
|
|
1241
|
-
return term;
|
|
1242
|
-
}
|
|
1243
|
-
};
|
|
1244
|
-
}
|
|
1245
|
-
|
|
1246
|
-
// src/persistence/durability/segment/load-partition.ts
|
|
1247
|
-
function requireAgreedSchema(agreed, schema, key) {
|
|
1248
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
1249
|
-
const existing = agreed[field];
|
|
1250
|
-
if (existing !== void 0 && existing !== type) {
|
|
1251
|
-
throw new NarsilError(
|
|
1252
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1253
|
-
`The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
|
|
1254
|
-
{ key, field, existing, found: type }
|
|
1255
|
-
);
|
|
1256
|
-
}
|
|
1257
|
-
}
|
|
1258
|
-
}
|
|
1259
|
-
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
1260
|
-
manager.beginBatchRemove();
|
|
1261
|
-
try {
|
|
1262
|
-
for (const docId of contents.tombstones) {
|
|
1263
|
-
if (manager.has(docId)) manager.remove(docId);
|
|
1264
|
-
}
|
|
1265
|
-
const ids = segment.arrays.idTable;
|
|
1266
|
-
for (let ordinal = 0; ordinal < ids.count; ordinal++) {
|
|
1267
|
-
const docId = ids.idAt(ordinal);
|
|
1268
|
-
if (manager.has(docId)) manager.remove(docId);
|
|
1269
|
-
}
|
|
1270
|
-
} finally {
|
|
1271
|
-
manager.endBatchRemove();
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
function yieldToEventLoop() {
|
|
1275
|
-
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
1276
|
-
}
|
|
1277
|
-
function removeFromPartition(manager, partitionId, docIds) {
|
|
1278
|
-
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
1279
|
-
composite.beginBatch();
|
|
1280
|
-
try {
|
|
1281
|
-
for (const docId of docIds) {
|
|
1282
|
-
if (!composite.has(docId)) continue;
|
|
1283
|
-
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
1284
|
-
docPartitionMap.delete(docId);
|
|
1285
|
-
}
|
|
1286
|
-
} finally {
|
|
1287
|
-
composite.endBatch();
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
1291
|
-
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-JYHFTVPS.mjs');
|
|
1292
|
-
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
1293
|
-
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1294
|
-
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
1295
|
-
const segment = createPagedFrozenSegment(snapshot);
|
|
1296
|
-
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
1297
|
-
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
1298
|
-
if (snapshot.documentCount > 0) {
|
|
1299
|
-
manager.attachFrozenSegment(partitionId, segment);
|
|
1300
|
-
return segment;
|
|
1301
|
-
}
|
|
1302
|
-
segment.paged?.close();
|
|
1303
|
-
return null;
|
|
1304
|
-
}
|
|
1305
|
-
function documentsHeldBefore(manager, partitionId) {
|
|
1306
|
-
const held = manager.partitionAt(partitionId);
|
|
1307
|
-
return held === void 0 ? [] : [...held.docIds()];
|
|
1308
|
-
}
|
|
1309
|
-
function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
|
|
1310
|
-
const absent = heldBefore.filter(
|
|
1311
|
-
(docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
|
|
1312
|
-
);
|
|
1313
|
-
if (absent.length === 0) return;
|
|
1314
|
-
manager.beginBatchRemove();
|
|
1315
|
-
try {
|
|
1316
|
-
for (const docId of absent) manager.remove(docId);
|
|
1317
|
-
} finally {
|
|
1318
|
-
manager.endBatchRemove();
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
async function loadPartitionSegments(store, partition, manager, options) {
|
|
1322
|
-
const heldBefore = documentsHeldBefore(manager, partition.partitionId);
|
|
1323
|
-
const loaded = await attachPartitionSegments(store, partition, manager, options);
|
|
1324
|
-
removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
|
|
1325
|
-
}
|
|
1326
|
-
async function attachPartitionSegments(store, partition, manager, options) {
|
|
1327
|
-
const loaded = [];
|
|
1328
|
-
const { directory } = store;
|
|
1329
|
-
for (const ref of partition.segments) {
|
|
1330
|
-
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
1331
|
-
const { isPagedSegmentFile } = await import('./module-JYHFTVPS.mjs');
|
|
1332
|
-
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
1333
|
-
const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
1334
|
-
if (paged !== null) loaded.push(paged);
|
|
1335
|
-
await yieldToEventLoop();
|
|
1336
|
-
continue;
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
const contents = await readSegmentFile(store, ref);
|
|
1340
|
-
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
1341
|
-
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
1342
|
-
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
1343
|
-
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
1344
|
-
options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
|
|
1345
|
-
if (contents.arrays.documentCount > 0) {
|
|
1346
|
-
manager.attachFrozenSegment(partition.partitionId, segment);
|
|
1347
|
-
loaded.push(segment);
|
|
1348
|
-
}
|
|
1349
|
-
await yieldToEventLoop();
|
|
1350
|
-
}
|
|
1351
|
-
return loaded;
|
|
1352
|
-
}
|
|
1353
|
-
|
|
1354
|
-
// src/persistence/durability/segment/load.ts
|
|
1355
|
-
async function readSegmentManifest(store, indexName) {
|
|
1356
|
-
const bytes = await store.read(manifestKey(indexName));
|
|
1357
|
-
if (bytes === null) return null;
|
|
1358
|
-
return decodeSegmentManifest(bytes, indexName);
|
|
1359
|
-
}
|
|
1360
|
-
async function readCurrentManifest(store, indexName) {
|
|
1361
|
-
const decoded = await readSegmentManifest(store, indexName);
|
|
1362
|
-
if (decoded === null) return null;
|
|
1363
|
-
if (decoded.kind === "current") return decoded.manifest;
|
|
1364
|
-
throw new NarsilError(
|
|
1365
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1366
|
-
`Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
|
|
1367
|
-
{ indexName }
|
|
1368
|
-
);
|
|
1369
|
-
}
|
|
1370
|
-
function growPartitionsTo(manager, highestPartitionId) {
|
|
1371
|
-
while (manager.partitionCount <= highestPartitionId) {
|
|
1372
|
-
const before = manager.partitionCount;
|
|
1373
|
-
try {
|
|
1374
|
-
manager.addPartition();
|
|
1375
|
-
} catch {
|
|
1376
|
-
break;
|
|
1377
|
-
}
|
|
1378
|
-
if (manager.partitionCount === before) break;
|
|
1379
|
-
}
|
|
1380
|
-
}
|
|
1381
|
-
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
1382
|
-
let highestPartitionId = manifest.partitions.length - 1;
|
|
1383
|
-
for (const partition of manifest.partitions) {
|
|
1384
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
1385
|
-
}
|
|
1386
|
-
growPartitionsTo(deps.manager, highestPartitionId);
|
|
1387
|
-
const { directory } = store;
|
|
1388
|
-
const opensVectorsOnDemand = manifest.vectors.some(
|
|
1389
|
-
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
1390
|
-
);
|
|
1391
|
-
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
1392
|
-
const { startReadingOpeningSectionsAhead } = await import('./module-JYHFTVPS.mjs');
|
|
1393
|
-
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
1394
|
-
}
|
|
1395
|
-
for (const partition of manifest.partitions) {
|
|
1396
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
1397
|
-
throw new NarsilError(
|
|
1398
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1399
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
1400
|
-
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
1401
|
-
);
|
|
1402
|
-
}
|
|
1403
|
-
await loadPartitionSegments(store, partition, deps.manager, {
|
|
1404
|
-
schema: manifest.schema,
|
|
1405
|
-
decodesEveryDocument,
|
|
1406
|
-
writeRecords: deps.writeRecords ?? null
|
|
1407
|
-
});
|
|
1408
|
-
}
|
|
1409
|
-
for (const vector of manifest.vectors) {
|
|
1410
|
-
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
1411
|
-
if (vectorIndex === void 0) continue;
|
|
1412
|
-
await loadVectorField(
|
|
1413
|
-
store,
|
|
1414
|
-
vector,
|
|
1415
|
-
vectorIndex,
|
|
1416
|
-
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
1417
|
-
);
|
|
1418
|
-
}
|
|
1419
|
-
return manifest.checkpoint;
|
|
1420
|
-
}
|
|
1421
|
-
async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
1422
|
-
for (const key of await store.list(segmentsPrefix(indexName))) {
|
|
1423
|
-
if (!referenced.has(key)) await store.remove(key);
|
|
1424
|
-
}
|
|
1425
|
-
}
|
|
1426
|
-
|
|
1427
|
-
export { SINGLE_NODE_PRIMARY_TERM, createSeqOwner, deserializePayloadV1, dropStoredVectorValues, durableMetadataConfigOf, growPartitionsTo, indexMetadataOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };
|