@delali/narsil 0.3.1-next.99 → 0.4.1-next.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +30 -11
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +404 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -0,0 +1,797 @@
|
|
|
1
|
+
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-C4ZMXXTL.mjs';
|
|
2
|
+
import { encode } from './chunk-S5QMWECL.mjs';
|
|
3
|
+
import { createFixedBuffer, createVectorFileReader } from './chunk-R724DJSG.mjs';
|
|
4
|
+
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-TKKXPKEO.mjs';
|
|
5
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-ASZLNVWW.mjs';
|
|
7
|
+
import { ErrorCodes, NarsilError } from './chunk-ZKOQR7EA.mjs';
|
|
8
|
+
|
|
9
|
+
// src/persistence/durability/segment/store.ts
|
|
10
|
+
function directoryCheckpointStore(directory) {
|
|
11
|
+
const store = {
|
|
12
|
+
read: (key) => directory.read(key),
|
|
13
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
14
|
+
remove: (key) => directory.remove(key),
|
|
15
|
+
list: (prefix) => directory.list(prefix),
|
|
16
|
+
pathOf: (key) => directory.pathOf(key),
|
|
17
|
+
directory
|
|
18
|
+
};
|
|
19
|
+
const byteLength = directory.byteLength;
|
|
20
|
+
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
21
|
+
return store;
|
|
22
|
+
}
|
|
23
|
+
function joined(chunks) {
|
|
24
|
+
if (chunks.length === 1) return chunks[0];
|
|
25
|
+
let total = 0;
|
|
26
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
27
|
+
const bytes = new Uint8Array(total);
|
|
28
|
+
let offset = 0;
|
|
29
|
+
for (const chunk of chunks) {
|
|
30
|
+
bytes.set(chunk, offset);
|
|
31
|
+
offset += chunk.byteLength;
|
|
32
|
+
}
|
|
33
|
+
return bytes;
|
|
34
|
+
}
|
|
35
|
+
function adapterFailure(code, verb, key, error) {
|
|
36
|
+
if (error instanceof NarsilError) return error;
|
|
37
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
38
|
+
return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
|
|
39
|
+
}
|
|
40
|
+
async function guarded(run, code, verb, key) {
|
|
41
|
+
try {
|
|
42
|
+
return await run();
|
|
43
|
+
} catch (error) {
|
|
44
|
+
throw adapterFailure(code, verb, key, error);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function adapterCheckpointStore(adapter) {
|
|
48
|
+
return {
|
|
49
|
+
read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
|
|
50
|
+
write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
|
|
51
|
+
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
52
|
+
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
async function requireListedSize(store, key, expected) {
|
|
56
|
+
const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
|
|
57
|
+
if (length === null) missingFile(expected.what, key);
|
|
58
|
+
if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
|
|
59
|
+
}
|
|
60
|
+
async function readListedFile(store, key, expected) {
|
|
61
|
+
await requireListedSize(store, key, expected);
|
|
62
|
+
const bytes = await store.read(key);
|
|
63
|
+
if (bytes === null) missingFile(expected.what, key);
|
|
64
|
+
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
65
|
+
return bytes;
|
|
66
|
+
}
|
|
67
|
+
function missingFile(what, key) {
|
|
68
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
|
|
69
|
+
key
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
function requireListedLength(what, key, found, listed) {
|
|
73
|
+
if (found === listed) return;
|
|
74
|
+
throw new NarsilError(
|
|
75
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
76
|
+
`The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
|
|
77
|
+
{ key, bytes: found, listed }
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
function requireListedChecksum(what, key, found, listed) {
|
|
81
|
+
if (found === listed) return;
|
|
82
|
+
throw new NarsilError(
|
|
83
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
84
|
+
`The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
|
|
85
|
+
{ key, checksum: found, listed }
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// src/persistence/paged/format.ts
|
|
90
|
+
var PAGED_SEGMENT_ENVELOPE_VERSION = 6;
|
|
91
|
+
var PAGED_SEGMENT_HEAD_VERSION = 1;
|
|
92
|
+
var SEGMENT_PAGE_BYTES = 65536;
|
|
93
|
+
var RECORD_PAGE_BYTES = 16384;
|
|
94
|
+
var SECTION_ALIGNMENT_BYTES = 4;
|
|
95
|
+
var PAYLOAD_PREFIX_BYTES = 8;
|
|
96
|
+
var POSTING_WIDTH_BYTES = 8;
|
|
97
|
+
var POSITION_WIDTH_BYTES = 4;
|
|
98
|
+
var SMALLEST_BLOCK_POSTINGS = 128;
|
|
99
|
+
var DOCUMENT_BLOCK_MIN_RAW_BYTES = 65536;
|
|
100
|
+
var MAX_DOCUMENT_BLOCK_RAW_BYTES = 16842752;
|
|
101
|
+
var ID_SAMPLE_INTERVAL = 256;
|
|
102
|
+
var BLOCK_SUMMARY_BYTES = 22;
|
|
103
|
+
var DOCUMENT_BLOCK_BYTES = 20;
|
|
104
|
+
function loadFailed(message, details) {
|
|
105
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
|
|
106
|
+
}
|
|
107
|
+
function isByteCount(value) {
|
|
108
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
109
|
+
}
|
|
110
|
+
function decodeSection(raw, what, allowedPageBytes) {
|
|
111
|
+
if (raw === null || typeof raw !== "object") loadFailed(`The ${what} lists a section that is not a map`);
|
|
112
|
+
const entry = raw;
|
|
113
|
+
const { name, offset, length, pageBytes, checksums } = entry;
|
|
114
|
+
if (typeof name !== "string" || name.length === 0) loadFailed(`The ${what} lists a section without a name`);
|
|
115
|
+
if (!isByteCount(offset) || !isByteCount(length)) {
|
|
116
|
+
loadFailed(`The ${what} gives section "${name}" an invalid place`, { section: name, offset, length });
|
|
117
|
+
}
|
|
118
|
+
if (typeof pageBytes !== "number" || !allowedPageBytes.has(pageBytes)) {
|
|
119
|
+
loadFailed(`The ${what} gives section "${name}" a page size of ${String(pageBytes)}`, { section: name, pageBytes });
|
|
120
|
+
}
|
|
121
|
+
if (!(checksums instanceof Uint8Array)) loadFailed(`The ${what} gives section "${name}" no checksums`);
|
|
122
|
+
if (checksums.byteLength !== Math.ceil(length / pageBytes) * 4) {
|
|
123
|
+
loadFailed(`The ${what} gives section "${name}" ${checksums.byteLength} bytes of checksums`, {
|
|
124
|
+
section: name,
|
|
125
|
+
checksumBytes: checksums.byteLength
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
return { name, offset, length, pageBytes, checksums };
|
|
129
|
+
}
|
|
130
|
+
function decodeSections(raw, payloadLength, what, allowedPageBytes) {
|
|
131
|
+
if (!Array.isArray(raw)) loadFailed(`The ${what} holds no section list`);
|
|
132
|
+
const sections = raw.map((entry) => decodeSection(entry, what, allowedPageBytes));
|
|
133
|
+
const names = /* @__PURE__ */ new Set();
|
|
134
|
+
for (const section of sections) {
|
|
135
|
+
if (names.has(section.name)) loadFailed(`The ${what} repeats section "${section.name}"`, { section: section.name });
|
|
136
|
+
names.add(section.name);
|
|
137
|
+
if (section.offset + section.length > payloadLength) {
|
|
138
|
+
loadFailed(`The ${what} ends section "${section.name}" outside the payload`, { section: section.name });
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
const byOffset = [...sections].sort((left, right) => left.offset - right.offset);
|
|
142
|
+
for (let at = 1; at < byOffset.length; at++) {
|
|
143
|
+
const before = byOffset[at - 1];
|
|
144
|
+
if (before.offset + before.length > byOffset[at].offset) {
|
|
145
|
+
loadFailed(`The ${what} overlaps section "${before.name}" with "${byOffset[at].name}"`, {
|
|
146
|
+
section: before.name
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return sections;
|
|
151
|
+
}
|
|
152
|
+
var SEGMENT_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
|
|
153
|
+
function isStringRecord(value) {
|
|
154
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
155
|
+
return Object.values(value).every((entry) => typeof entry === "string");
|
|
156
|
+
}
|
|
157
|
+
function isStringList(value) {
|
|
158
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
159
|
+
}
|
|
160
|
+
function decodePagedSegmentHead(head, payloadLength, what) {
|
|
161
|
+
let raw;
|
|
162
|
+
try {
|
|
163
|
+
raw = decode(head);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
loadFailed(`The head of ${what} is not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
|
|
166
|
+
}
|
|
167
|
+
if (raw === null || typeof raw !== "object") loadFailed(`The head of ${what} is not a map`);
|
|
168
|
+
const entry = raw;
|
|
169
|
+
if (entry.v !== PAGED_SEGMENT_HEAD_VERSION) {
|
|
170
|
+
throw new NarsilError(
|
|
171
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
172
|
+
`The head of ${what} was written in layout ${String(entry.v)}, and this build reads layout ${PAGED_SEGMENT_HEAD_VERSION} alone`,
|
|
173
|
+
{ version: entry.v }
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
if (!isStringRecord(entry.schema)) loadFailed(`The head of ${what} holds no schema`);
|
|
177
|
+
if (!isStringList(entry.tombstones)) loadFailed(`The head of ${what} holds no tombstone list`);
|
|
178
|
+
if (!isStringList(entry.fieldNames)) loadFailed(`The head of ${what} holds no field names`);
|
|
179
|
+
const { documentCount } = entry;
|
|
180
|
+
if (typeof documentCount !== "number" || !Number.isInteger(documentCount) || documentCount < 0) {
|
|
181
|
+
loadFailed(`The head of ${what} holds an invalid document count`, { documentCount });
|
|
182
|
+
}
|
|
183
|
+
if (documentCount > 4294967295) loadFailed(`The head of ${what} holds a document count above uint32`);
|
|
184
|
+
return {
|
|
185
|
+
v: PAGED_SEGMENT_HEAD_VERSION,
|
|
186
|
+
schema: entry.schema,
|
|
187
|
+
tombstones: entry.tombstones,
|
|
188
|
+
documentCount,
|
|
189
|
+
fieldNames: entry.fieldNames,
|
|
190
|
+
sections: decodeSections(entry.sections, payloadLength, `head of ${what}`, SEGMENT_PAGE_SIZES)
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function alignedUp(offset) {
|
|
194
|
+
const over = offset % SECTION_ALIGNMENT_BYTES;
|
|
195
|
+
return over === 0 ? offset : offset + SECTION_ALIGNMENT_BYTES - over;
|
|
196
|
+
}
|
|
197
|
+
function placeSections(planned, sectionsStart) {
|
|
198
|
+
let cursor = sectionsStart;
|
|
199
|
+
return planned.map((section) => {
|
|
200
|
+
const offset = alignedUp(cursor);
|
|
201
|
+
cursor = offset + section.length;
|
|
202
|
+
return {
|
|
203
|
+
name: section.name,
|
|
204
|
+
offset,
|
|
205
|
+
length: section.length,
|
|
206
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
207
|
+
checksums: section.checksums
|
|
208
|
+
};
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
function encodePagedSegmentHead(fields, planned) {
|
|
212
|
+
let headLength = 0;
|
|
213
|
+
for (; ; ) {
|
|
214
|
+
const sectionsStart = alignedUp(PAYLOAD_PREFIX_BYTES + headLength);
|
|
215
|
+
const sections = placeSections(planned, sectionsStart);
|
|
216
|
+
const head = encode({ v: PAGED_SEGMENT_HEAD_VERSION, ...fields, sections });
|
|
217
|
+
if (head.byteLength !== headLength) {
|
|
218
|
+
headLength = head.byteLength;
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const prefix = new Uint8Array(PAYLOAD_PREFIX_BYTES);
|
|
222
|
+
const view = new DataView(prefix.buffer);
|
|
223
|
+
view.setUint32(0, head.byteLength, true);
|
|
224
|
+
view.setUint32(4, crc32(head), true);
|
|
225
|
+
const last = sections[sections.length - 1];
|
|
226
|
+
const payloadLength = last === void 0 ? sectionsStart : last.offset + last.length;
|
|
227
|
+
return { head, prefix, sections, payloadLength };
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
function sectionNamed(sections, name) {
|
|
231
|
+
for (const section of sections) {
|
|
232
|
+
if (section.name === name) return section;
|
|
233
|
+
}
|
|
234
|
+
return void 0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// src/persistence/durability/segment/manifest-encode.ts
|
|
238
|
+
function fileRefOnTheWire(ref) {
|
|
239
|
+
return {
|
|
240
|
+
key: ref.key,
|
|
241
|
+
bytes: ref.bytes,
|
|
242
|
+
checksum: ref.checksum,
|
|
243
|
+
...ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum }
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
function encodeSegmentManifest(manifest) {
|
|
247
|
+
const payload = encode({
|
|
248
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
249
|
+
schema: manifest.schema,
|
|
250
|
+
language: manifest.language,
|
|
251
|
+
...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
|
|
252
|
+
checkpoint: manifest.checkpoint.map((c) => ({
|
|
253
|
+
partitionId: c.partitionId,
|
|
254
|
+
lastSeqNo: c.lastSeqNo,
|
|
255
|
+
primaryTerm: c.primaryTerm,
|
|
256
|
+
...c.termHistory === void 0 ? {} : {
|
|
257
|
+
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
258
|
+
}
|
|
259
|
+
})),
|
|
260
|
+
partitions: manifest.partitions.map((p) => ({
|
|
261
|
+
partitionId: p.partitionId,
|
|
262
|
+
nextSegmentId: p.nextSegmentId,
|
|
263
|
+
segments: p.segments.map((s) => ({
|
|
264
|
+
id: s.id,
|
|
265
|
+
key: s.key,
|
|
266
|
+
docCount: s.docCount,
|
|
267
|
+
tombstoneCount: s.tombstoneCount,
|
|
268
|
+
bytes: s.bytes,
|
|
269
|
+
checksum: s.checksum
|
|
270
|
+
}))
|
|
271
|
+
})),
|
|
272
|
+
vectors: manifest.vectors.map((v) => ({
|
|
273
|
+
fieldPath: v.fieldPath,
|
|
274
|
+
nextFileId: v.nextFileId,
|
|
275
|
+
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
276
|
+
graphGeneration: v.graphGeneration,
|
|
277
|
+
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
278
|
+
}))
|
|
279
|
+
});
|
|
280
|
+
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// src/persistence/paged/native-file.ts
|
|
284
|
+
var PAGE_CORRUPT = -9;
|
|
285
|
+
function requireNativePage(status, path, section, page) {
|
|
286
|
+
if (status === 0) return;
|
|
287
|
+
failNativePage(status, path, section, page);
|
|
288
|
+
}
|
|
289
|
+
function failNativePage(status, path, section, page) {
|
|
290
|
+
const code = status === PAGE_CORRUPT ? ErrorCodes.PERSISTENCE_CRC_MISMATCH : ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
291
|
+
throw new NarsilError(code, `The core cannot read page ${page} of section "${section}" of "${path}"`, {
|
|
292
|
+
path,
|
|
293
|
+
section,
|
|
294
|
+
page: Number(page),
|
|
295
|
+
status
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
function openNativePagedFile(path) {
|
|
299
|
+
const core = nativeSearchCoreIsEnabled() ? loadNativeCore() : null;
|
|
300
|
+
if (core === null) return null;
|
|
301
|
+
const handle = core.openPagedFile(path);
|
|
302
|
+
if (typeof handle === "number") {
|
|
303
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The core cannot open "${path}" for reading`, {
|
|
304
|
+
path,
|
|
305
|
+
status: handle
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
return { core, handle };
|
|
309
|
+
}
|
|
310
|
+
function nativeSection(file, section) {
|
|
311
|
+
if (file === null) return null;
|
|
312
|
+
return {
|
|
313
|
+
file: file.handle,
|
|
314
|
+
offset: BigInt(HEADER_SIZE + section.offset),
|
|
315
|
+
length: BigInt(section.length),
|
|
316
|
+
pageBytes: section.pageBytes,
|
|
317
|
+
checksums: section.checksums
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// src/persistence/paged/constants.ts
|
|
322
|
+
var THREAD_READ_CACHE_BYTES = 134217728;
|
|
323
|
+
var NATIVE_PAGE_CACHE_SHARE = 0.25;
|
|
324
|
+
var NATIVE_ASYNC_PAGE_CACHES = 4;
|
|
325
|
+
var MERGE_READ_CACHE_BYTES = 33554432;
|
|
326
|
+
var MERGED_SEGMENT_TARGET_BYTES = 1073741824;
|
|
327
|
+
var MERGED_SEGMENT_GROWTH_ALLOWANCE = 1.01;
|
|
328
|
+
var SECTION_WRITE_BUFFER_BYTES = 1048576;
|
|
329
|
+
var REMOVAL_BUCKET_WRITE_BUFFER_BYTES = 262144;
|
|
330
|
+
var REMOVAL_BUCKET_TARGET_BYTES = 67108864;
|
|
331
|
+
var LARGEST_REMOVAL_BUCKET_COUNT = 256;
|
|
332
|
+
var WRITER_YIELD_INTERVAL_TOKENS = 4096;
|
|
333
|
+
var FILE_ASSEMBLY_CHUNK_BYTES = 4194304;
|
|
334
|
+
var LARGEST_CACHED_ENTRY_SHARE = 0.25;
|
|
335
|
+
var MAX_THREAD_FILE_NUMBER = 524288;
|
|
336
|
+
var LIST_VIEWS_HELD_PER_SEGMENT = 16384;
|
|
337
|
+
var COLD_LOAD_WAIT_WITHOUT_SEARCH_MS = 400;
|
|
338
|
+
var TOMBSTONES_BETWEEN_YIELDS = 4096;
|
|
339
|
+
var STABLE_HEAD_ATTEMPTS = 8;
|
|
340
|
+
var COLD_WALK_NODES_READ_AHEAD = 8;
|
|
341
|
+
var COLD_GRAPH_PART_VECTORS = 16384;
|
|
342
|
+
var READ_AHEAD_CHUNK_BYTES = 1048576;
|
|
343
|
+
var READS_AHEAD_AT_ONCE = 4;
|
|
344
|
+
var REWRITE_SPILL_BYTES = 134217728;
|
|
345
|
+
var POSTING_WRITE_ROWS = 1024;
|
|
346
|
+
var POSTING_WRITE_POSITIONS = 4096;
|
|
347
|
+
|
|
348
|
+
// src/persistence/paged/page-cache.ts
|
|
349
|
+
var SLOT_BYTES = 65536;
|
|
350
|
+
var SLOT_METADATA_BYTES = 3 * BigUint64Array.BYTES_PER_ELEMENT + 2 * Uint32Array.BYTES_PER_ELEMENT;
|
|
351
|
+
function createPageCache(capacity, shared = false) {
|
|
352
|
+
const slots = Math.floor(capacity / (SLOT_BYTES + SLOT_METADATA_BYTES));
|
|
353
|
+
const allocate = (bytes) => shared ? createFixedBuffer(bytes) : new ArrayBuffer(bytes);
|
|
354
|
+
return {
|
|
355
|
+
bytes: new Uint8Array(allocate(slots * SLOT_BYTES)),
|
|
356
|
+
files: new BigUint64Array(allocate(slots * BigUint64Array.BYTES_PER_ELEMENT)),
|
|
357
|
+
offsets: new BigUint64Array(allocate(slots * BigUint64Array.BYTES_PER_ELEMENT)),
|
|
358
|
+
pages: new BigUint64Array(allocate(slots * BigUint64Array.BYTES_PER_ELEMENT)),
|
|
359
|
+
lengths: new Uint32Array(allocate(slots * Uint32Array.BYTES_PER_ELEMENT)),
|
|
360
|
+
checksums: new Uint32Array(allocate(slots * Uint32Array.BYTES_PER_ELEMENT))
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
function pageCacheBytes(cache) {
|
|
364
|
+
return cache.bytes.byteLength + cache.files.byteLength + cache.offsets.byteLength + cache.pages.byteLength + cache.lengths.byteLength + cache.checksums.byteLength;
|
|
365
|
+
}
|
|
366
|
+
function pageCacheBytesByFile(cache) {
|
|
367
|
+
const bytes = /* @__PURE__ */ new Map();
|
|
368
|
+
for (const identity of cache.files) {
|
|
369
|
+
if (identity !== 0n) bytes.set(identity, (bytes.get(identity) ?? 0) + SLOT_BYTES + SLOT_METADATA_BYTES);
|
|
370
|
+
}
|
|
371
|
+
return bytes;
|
|
372
|
+
}
|
|
373
|
+
function pageCacheFileBytes(cache, identity) {
|
|
374
|
+
let bytes = 0;
|
|
375
|
+
for (const held of cache.files) {
|
|
376
|
+
if (held === identity) bytes += SLOT_BYTES + SLOT_METADATA_BYTES;
|
|
377
|
+
}
|
|
378
|
+
return bytes;
|
|
379
|
+
}
|
|
380
|
+
function releasePageFile(cache, identity) {
|
|
381
|
+
for (let at = 0; at < cache.files.length; at++) {
|
|
382
|
+
if (cache.files[at] === identity) cache.files[at] = 0n;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// src/persistence/paged/async-page-cache.ts
|
|
387
|
+
function createAsyncPageCaches(capacity, account) {
|
|
388
|
+
const leases = /* @__PURE__ */ new Set();
|
|
389
|
+
function release(lease) {
|
|
390
|
+
lease.active = false;
|
|
391
|
+
if (lease.discard) {
|
|
392
|
+
leases.delete(lease);
|
|
393
|
+
account(-pageCacheBytes(lease.cache));
|
|
394
|
+
} else {
|
|
395
|
+
for (const identity of lease.releasedFiles) releasePageFile(lease.cache, identity);
|
|
396
|
+
lease.bytesByFile = pageCacheBytesByFile(lease.cache);
|
|
397
|
+
}
|
|
398
|
+
lease.releasedFiles.clear();
|
|
399
|
+
}
|
|
400
|
+
return {
|
|
401
|
+
async withCache(read) {
|
|
402
|
+
let lease;
|
|
403
|
+
for (const available of leases) {
|
|
404
|
+
if (!available.active) {
|
|
405
|
+
lease = available;
|
|
406
|
+
break;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
if (lease === void 0) {
|
|
410
|
+
const retained = leases.size < NATIVE_ASYNC_PAGE_CACHES;
|
|
411
|
+
const cache = createPageCache(retained ? capacity / NATIVE_ASYNC_PAGE_CACHES : 0, true);
|
|
412
|
+
lease = { cache, active: false, discard: !retained, releasedFiles: /* @__PURE__ */ new Set(), bytesByFile: /* @__PURE__ */ new Map() };
|
|
413
|
+
leases.add(lease);
|
|
414
|
+
account(pageCacheBytes(cache));
|
|
415
|
+
}
|
|
416
|
+
lease.active = true;
|
|
417
|
+
try {
|
|
418
|
+
return await read(lease.cache);
|
|
419
|
+
} finally {
|
|
420
|
+
release(lease);
|
|
421
|
+
}
|
|
422
|
+
},
|
|
423
|
+
releaseFile(identity) {
|
|
424
|
+
for (const lease of leases) {
|
|
425
|
+
lease.bytesByFile.delete(identity);
|
|
426
|
+
if (lease.active) lease.releasedFiles.add(identity);
|
|
427
|
+
else releasePageFile(lease.cache, identity);
|
|
428
|
+
}
|
|
429
|
+
},
|
|
430
|
+
bytesOfFile(identity) {
|
|
431
|
+
let bytes = 0;
|
|
432
|
+
for (const lease of leases) bytes += lease.bytesByFile.get(identity) ?? 0;
|
|
433
|
+
return bytes;
|
|
434
|
+
},
|
|
435
|
+
clear() {
|
|
436
|
+
for (const lease of leases) {
|
|
437
|
+
lease.discard = true;
|
|
438
|
+
if (!lease.active) release(lease);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// src/persistence/paged/read-cache.ts
|
|
445
|
+
var FILE_NUMBER_SCALE = 2 ** 34;
|
|
446
|
+
var ENTRY_NUMBER_SCALE = 4;
|
|
447
|
+
var CACHED_PAGE = 0;
|
|
448
|
+
var CACHED_LIST = 1;
|
|
449
|
+
var CACHED_POSITIONS = 2;
|
|
450
|
+
var CACHED_DOCUMENT_BLOCK = 3;
|
|
451
|
+
function createReadCache(capacity) {
|
|
452
|
+
const entries = /* @__PURE__ */ new Map();
|
|
453
|
+
const perFile = /* @__PURE__ */ new Map();
|
|
454
|
+
const freedFileNumbers = [];
|
|
455
|
+
let held = 0;
|
|
456
|
+
let pages = null;
|
|
457
|
+
let reserved = 0;
|
|
458
|
+
let nextFileNumber = 1;
|
|
459
|
+
const asyncPages = createAsyncPageCaches(capacity * NATIVE_PAGE_CACHE_SHARE, (bytes) => {
|
|
460
|
+
reserved += bytes;
|
|
461
|
+
evictDownTo(capacity - reserved);
|
|
462
|
+
});
|
|
463
|
+
function drop(key, entry) {
|
|
464
|
+
entries.delete(key);
|
|
465
|
+
held -= entry.bytes;
|
|
466
|
+
const remaining = (perFile.get(entry.fileNumber) ?? 0) - entry.bytes;
|
|
467
|
+
if (remaining <= 0) perFile.delete(entry.fileNumber);
|
|
468
|
+
else perFile.set(entry.fileNumber, remaining);
|
|
469
|
+
}
|
|
470
|
+
function evictDownTo(limit) {
|
|
471
|
+
for (const [key, entry] of entries) {
|
|
472
|
+
if (held <= limit) return;
|
|
473
|
+
if (entry.referenced) {
|
|
474
|
+
entry.referenced = false;
|
|
475
|
+
entries.delete(key);
|
|
476
|
+
entries.set(key, entry);
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
479
|
+
drop(key, entry);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
function clear() {
|
|
483
|
+
entries.clear();
|
|
484
|
+
perFile.clear();
|
|
485
|
+
held = 0;
|
|
486
|
+
if (pages !== null) reserved -= pageCacheBytes(pages);
|
|
487
|
+
pages = null;
|
|
488
|
+
asyncPages.clear();
|
|
489
|
+
}
|
|
490
|
+
return {
|
|
491
|
+
get bytes() {
|
|
492
|
+
return held + reserved;
|
|
493
|
+
},
|
|
494
|
+
capacity,
|
|
495
|
+
pageCache() {
|
|
496
|
+
if (pages === null) {
|
|
497
|
+
const created = createPageCache(capacity * NATIVE_PAGE_CACHE_SHARE);
|
|
498
|
+
reserved += pageCacheBytes(created);
|
|
499
|
+
evictDownTo(capacity - reserved);
|
|
500
|
+
pages = created;
|
|
501
|
+
}
|
|
502
|
+
return pages;
|
|
503
|
+
},
|
|
504
|
+
withPageCache: asyncPages.withCache,
|
|
505
|
+
releasePageFile(identity) {
|
|
506
|
+
if (pages !== null) releasePageFile(pages, identity);
|
|
507
|
+
asyncPages.releaseFile(identity);
|
|
508
|
+
},
|
|
509
|
+
fileNumber() {
|
|
510
|
+
const freed = freedFileNumbers.pop();
|
|
511
|
+
if (freed !== void 0) return freed;
|
|
512
|
+
if (nextFileNumber >= MAX_THREAD_FILE_NUMBER) {
|
|
513
|
+
throw new NarsilError(
|
|
514
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
515
|
+
`This thread holds ${nextFileNumber} open segment files, which is the most that its read cache can tell apart`,
|
|
516
|
+
{ limit: MAX_THREAD_FILE_NUMBER }
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
const number = nextFileNumber;
|
|
520
|
+
nextFileNumber += 1;
|
|
521
|
+
return number;
|
|
522
|
+
},
|
|
523
|
+
bytesOfFile(fileNumber) {
|
|
524
|
+
return perFile.get(fileNumber) ?? 0;
|
|
525
|
+
},
|
|
526
|
+
bytesOfPageFile(identity) {
|
|
527
|
+
return (pages === null ? 0 : pageCacheFileBytes(pages, identity)) + asyncPages.bytesOfFile(identity);
|
|
528
|
+
},
|
|
529
|
+
get(fileNumber, kind, entry) {
|
|
530
|
+
const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
|
|
531
|
+
const found = entries.get(key);
|
|
532
|
+
if (found === void 0) return void 0;
|
|
533
|
+
found.referenced = true;
|
|
534
|
+
return found.value;
|
|
535
|
+
},
|
|
536
|
+
put(fileNumber, kind, entry, value, bytes) {
|
|
537
|
+
if (bytes > capacity * LARGEST_CACHED_ENTRY_SHARE) return;
|
|
538
|
+
const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
|
|
539
|
+
const existing = entries.get(key);
|
|
540
|
+
if (existing !== void 0) drop(key, existing);
|
|
541
|
+
entries.set(key, { value, bytes, fileNumber, referenced: true });
|
|
542
|
+
held += bytes;
|
|
543
|
+
perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
|
|
544
|
+
evictDownTo(capacity - reserved);
|
|
545
|
+
},
|
|
546
|
+
releaseFile(fileNumber) {
|
|
547
|
+
freedFileNumbers.push(fileNumber);
|
|
548
|
+
if (!perFile.has(fileNumber)) return;
|
|
549
|
+
for (const [key, entry] of entries) {
|
|
550
|
+
if (entry.fileNumber === fileNumber) drop(key, entry);
|
|
551
|
+
}
|
|
552
|
+
},
|
|
553
|
+
clear
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
var threadCache = null;
|
|
557
|
+
function readCacheOfThisThread() {
|
|
558
|
+
threadCache ??= createReadCache(THREAD_READ_CACHE_BYTES);
|
|
559
|
+
return threadCache;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// src/persistence/paged/section-file.ts
|
|
563
|
+
var threadReader = null;
|
|
564
|
+
function fileReaderOfThisThread() {
|
|
565
|
+
threadReader ??= createVectorFileReader();
|
|
566
|
+
return threadReader;
|
|
567
|
+
}
|
|
568
|
+
var readersOfPath = /* @__PURE__ */ new Map();
|
|
569
|
+
function cachedFileBytes(file) {
|
|
570
|
+
if (file.closed) return 0;
|
|
571
|
+
return file.cache.bytesOfFile(file.fileNumber) + (file.native === null ? 0 : file.cache.bytesOfPageFile(file.native.handle.identity));
|
|
572
|
+
}
|
|
573
|
+
function openPagedFile(path, cache) {
|
|
574
|
+
const native = openNativePagedFile(path);
|
|
575
|
+
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
576
|
+
try {
|
|
577
|
+
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false, suspended: false };
|
|
578
|
+
} catch (error) {
|
|
579
|
+
native?.core.closePagedFile(native.handle);
|
|
580
|
+
stopReadingPath(path);
|
|
581
|
+
throw error;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
function stopReadingPath(path) {
|
|
585
|
+
const readers = (readersOfPath.get(path) ?? 1) - 1;
|
|
586
|
+
if (readers > 0) {
|
|
587
|
+
readersOfPath.set(path, readers);
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
readersOfPath.delete(path);
|
|
591
|
+
fileReaderOfThisThread().retainOnly(new Set(readersOfPath.keys()));
|
|
592
|
+
}
|
|
593
|
+
function closeNativeHandle(file) {
|
|
594
|
+
if (file.native !== null) file.cache.releasePageFile(file.native.handle.identity);
|
|
595
|
+
file.native?.core.closePagedFile(file.native.handle);
|
|
596
|
+
file.native = null;
|
|
597
|
+
}
|
|
598
|
+
function closePagedFile(file) {
|
|
599
|
+
if (file.closed) return;
|
|
600
|
+
file.closed = true;
|
|
601
|
+
closeNativeHandle(file);
|
|
602
|
+
file.cache.releaseFile(file.fileNumber);
|
|
603
|
+
if (file.suspended) return;
|
|
604
|
+
stopReadingPath(file.path);
|
|
605
|
+
}
|
|
606
|
+
function suspendPagedFile(file) {
|
|
607
|
+
if (file.closed || file.suspended) return;
|
|
608
|
+
file.suspended = true;
|
|
609
|
+
closeNativeHandle(file);
|
|
610
|
+
stopReadingPath(file.path);
|
|
611
|
+
}
|
|
612
|
+
function resumePagedFile(file, path) {
|
|
613
|
+
if (file.closed || !file.suspended) return;
|
|
614
|
+
file.suspended = false;
|
|
615
|
+
file.path = path;
|
|
616
|
+
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
617
|
+
file.native = openNativePagedFile(path);
|
|
618
|
+
}
|
|
619
|
+
function unreadable(path, offset, target) {
|
|
620
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The file "${path}" cannot be read at byte ${offset}`, {
|
|
621
|
+
path,
|
|
622
|
+
offset,
|
|
623
|
+
length: target.byteLength
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
function readFileRange(path, offset, target) {
|
|
627
|
+
if (!fileReaderOfThisThread().readInto(path, offset, target)) unreadable(path, offset, target);
|
|
628
|
+
}
|
|
629
|
+
function readFileRangeLater(path, offset, target) {
|
|
630
|
+
return fileReaderOfThisThread().readLater(path, offset, target);
|
|
631
|
+
}
|
|
632
|
+
function checksumOfPage(section, page) {
|
|
633
|
+
const at = page * 4;
|
|
634
|
+
const { checksums } = section;
|
|
635
|
+
return (checksums[at] | checksums[at + 1] << 8 | checksums[at + 2] << 16 | checksums[at + 3] << 24) >>> 0;
|
|
636
|
+
}
|
|
637
|
+
function requirePageChecksum(path, section, page, bytes) {
|
|
638
|
+
if (crc32(bytes) === checksumOfPage(section, page)) return;
|
|
639
|
+
throw new NarsilError(
|
|
640
|
+
ErrorCodes.PERSISTENCE_CRC_MISMATCH,
|
|
641
|
+
`Page ${page} of section "${section.name}" of the file "${path}" does not match its checksum`,
|
|
642
|
+
{ path, section: section.name, page }
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
function createSectionReader(file, section) {
|
|
646
|
+
const { pageBytes, length } = section;
|
|
647
|
+
const start = HEADER_SIZE + section.offset;
|
|
648
|
+
const fault = new BigUint64Array(3);
|
|
649
|
+
function pageAt(page) {
|
|
650
|
+
const entry = start + page * pageBytes;
|
|
651
|
+
const held = file.cache.get(file.fileNumber, CACHED_PAGE, entry);
|
|
652
|
+
if (held !== void 0) return held;
|
|
653
|
+
const bytes = new Uint8Array(Math.min(pageBytes, length - page * pageBytes));
|
|
654
|
+
const descriptor = nativeSection(file.native, section);
|
|
655
|
+
if (descriptor !== null && file.native !== null) {
|
|
656
|
+
const status = file.native.core.readPage(descriptor, file.cache.pageCache(), BigInt(page), bytes, fault);
|
|
657
|
+
requireNativePage(status, file.path, section.name, fault[2]);
|
|
658
|
+
} else {
|
|
659
|
+
readFileRange(file.path, entry, bytes);
|
|
660
|
+
requirePageChecksum(file.path, section, page, bytes);
|
|
661
|
+
}
|
|
662
|
+
file.cache.put(file.fileNumber, CACHED_PAGE, entry, bytes, bytes.byteLength);
|
|
663
|
+
return bytes;
|
|
664
|
+
}
|
|
665
|
+
async function fetchPage(page) {
|
|
666
|
+
const entry = start + page * pageBytes;
|
|
667
|
+
const size = Math.min(pageBytes, length - page * pageBytes);
|
|
668
|
+
const native = file.native;
|
|
669
|
+
const descriptor = nativeSection(native, section);
|
|
670
|
+
const bytes = new Uint8Array(native === null ? new ArrayBuffer(size) : createFixedBuffer(size));
|
|
671
|
+
if (native !== null && descriptor !== null) {
|
|
672
|
+
const failure = new BigUint64Array(createFixedBuffer(3 * BigUint64Array.BYTES_PER_ELEMENT));
|
|
673
|
+
const status = await native.core.readPageAsync(descriptor, null, BigInt(page), bytes, failure);
|
|
674
|
+
requireNativePage(status, file.path, section.name, failure[2]);
|
|
675
|
+
} else {
|
|
676
|
+
if (!await fileReaderOfThisThread().readLater(file.path, entry, bytes)) unreadable(file.path, entry, bytes);
|
|
677
|
+
requirePageChecksum(file.path, section, page, bytes);
|
|
678
|
+
}
|
|
679
|
+
if (!file.closed) file.cache.put(file.fileNumber, CACHED_PAGE, entry, bytes, bytes.byteLength);
|
|
680
|
+
}
|
|
681
|
+
function requireInside(position, size) {
|
|
682
|
+
if (!file.closed && Number.isSafeInteger(position) && Number.isSafeInteger(size) && position >= 0 && size >= 0 && position <= length && size <= length - position)
|
|
683
|
+
return;
|
|
684
|
+
throw new NarsilError(
|
|
685
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
686
|
+
`A read of ${size} bytes at ${position} ends outside section "${section.name}" of "${file.path}"`,
|
|
687
|
+
{ path: file.path, section: section.name, position, size, length }
|
|
688
|
+
);
|
|
689
|
+
}
|
|
690
|
+
function read(position, target) {
|
|
691
|
+
requireInside(position, target.byteLength);
|
|
692
|
+
let written = 0;
|
|
693
|
+
while (written < target.byteLength) {
|
|
694
|
+
const at = position + written;
|
|
695
|
+
const page = Math.floor(at / pageBytes);
|
|
696
|
+
const from = at - page * pageBytes;
|
|
697
|
+
const bytes = pageAt(page);
|
|
698
|
+
const take = Math.min(bytes.byteLength - from, target.byteLength - written);
|
|
699
|
+
target.set(bytes.subarray(from, from + take), written);
|
|
700
|
+
written += take;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return {
|
|
704
|
+
length,
|
|
705
|
+
get native() {
|
|
706
|
+
return nativeSection(file.native, section);
|
|
707
|
+
},
|
|
708
|
+
read,
|
|
709
|
+
fetchRows(positions, rowBytes) {
|
|
710
|
+
const wanted = /* @__PURE__ */ new Set();
|
|
711
|
+
for (const position of positions) {
|
|
712
|
+
requireInside(position, rowBytes);
|
|
713
|
+
const last = Math.floor((position + rowBytes - 1) / pageBytes);
|
|
714
|
+
for (let page = Math.floor(position / pageBytes); page <= last; page++) {
|
|
715
|
+
if (file.cache.get(file.fileNumber, CACHED_PAGE, start + page * pageBytes) === void 0) {
|
|
716
|
+
wanted.add(page);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
if (wanted.size === 0) return null;
|
|
721
|
+
return Promise.all(Array.from(wanted, fetchPage));
|
|
722
|
+
},
|
|
723
|
+
bytesAt(position, size) {
|
|
724
|
+
requireInside(position, size);
|
|
725
|
+
if (size === 0) return new Uint8Array(0);
|
|
726
|
+
const page = Math.floor(position / pageBytes);
|
|
727
|
+
const from = position - page * pageBytes;
|
|
728
|
+
if (from + size <= pageBytes) return pageAt(page).subarray(from, from + size);
|
|
729
|
+
const target = new Uint8Array(size);
|
|
730
|
+
read(position, target);
|
|
731
|
+
return target;
|
|
732
|
+
},
|
|
733
|
+
whole() {
|
|
734
|
+
const target = new Uint8Array(length);
|
|
735
|
+
read(0, target);
|
|
736
|
+
return target;
|
|
737
|
+
}
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
async function checkEverySectionPage(path, section, stillWanted) {
|
|
741
|
+
const native = openNativePagedFile(path);
|
|
742
|
+
if (native !== null) {
|
|
743
|
+
try {
|
|
744
|
+
const descriptor = nativeSection(native, section);
|
|
745
|
+
if (descriptor === null) return;
|
|
746
|
+
const output = new Uint8Array(createFixedBuffer(section.pageBytes));
|
|
747
|
+
const failure = new BigUint64Array(createFixedBuffer(3 * BigUint64Array.BYTES_PER_ELEMENT));
|
|
748
|
+
for (let page = 0; page * section.pageBytes < section.length; page++) {
|
|
749
|
+
if (!stillWanted()) return;
|
|
750
|
+
const status = await native.core.readPageAsync(descriptor, null, BigInt(page), output, failure);
|
|
751
|
+
requireNativePage(status, path, section.name, failure[2]);
|
|
752
|
+
}
|
|
753
|
+
} finally {
|
|
754
|
+
native.core.closePagedFile(native.handle);
|
|
755
|
+
}
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
const reader = fileReaderOfThisThread();
|
|
759
|
+
const start = HEADER_SIZE + section.offset;
|
|
760
|
+
for (let page = 0; page * section.pageBytes < section.length; page++) {
|
|
761
|
+
if (!stillWanted()) return;
|
|
762
|
+
const from = page * section.pageBytes;
|
|
763
|
+
const bytes = new Uint8Array(Math.min(section.pageBytes, section.length - from));
|
|
764
|
+
if (!await reader.readLater(path, start + from, bytes)) unreadable(path, start + from, bytes);
|
|
765
|
+
requirePageChecksum(path, section, page, bytes);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
function readWholeSectionChecked(path, section, shared) {
|
|
769
|
+
const target = new Uint8Array(shared ? new SharedArrayBuffer(section.length) : new ArrayBuffer(section.length));
|
|
770
|
+
const native = openNativePagedFile(path);
|
|
771
|
+
if (native !== null) {
|
|
772
|
+
try {
|
|
773
|
+
const descriptor = nativeSection(native, section);
|
|
774
|
+
if (descriptor === null) return target;
|
|
775
|
+
const failure = new BigUint64Array(3);
|
|
776
|
+
for (let page = 0; page * section.pageBytes < section.length; page++) {
|
|
777
|
+
const from = page * section.pageBytes;
|
|
778
|
+
const to = Math.min(from + section.pageBytes, section.length);
|
|
779
|
+
const status = native.core.readPage(descriptor, null, BigInt(page), target.subarray(from, to), failure);
|
|
780
|
+
requireNativePage(status, path, section.name, failure[2]);
|
|
781
|
+
}
|
|
782
|
+
} finally {
|
|
783
|
+
native.core.closePagedFile(native.handle);
|
|
784
|
+
}
|
|
785
|
+
return target;
|
|
786
|
+
}
|
|
787
|
+
const start = HEADER_SIZE + section.offset;
|
|
788
|
+
if (section.length > 0) readFileRange(path, start, target);
|
|
789
|
+
for (let page = 0; page * section.pageBytes < section.length; page++) {
|
|
790
|
+
const from = page * section.pageBytes;
|
|
791
|
+
const to = Math.min(from + section.pageBytes, section.length);
|
|
792
|
+
requirePageChecksum(path, section, page, target.subarray(from, to));
|
|
793
|
+
}
|
|
794
|
+
return target;
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
export { BLOCK_SUMMARY_BYTES, CACHED_DOCUMENT_BLOCK, CACHED_LIST, CACHED_POSITIONS, COLD_GRAPH_PART_VECTORS, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD, DOCUMENT_BLOCK_BYTES, DOCUMENT_BLOCK_MIN_RAW_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, ID_SAMPLE_INTERVAL, LARGEST_REMOVAL_BUCKET_COUNT, LIST_VIEWS_HELD_PER_SEGMENT, MAX_DOCUMENT_BLOCK_RAW_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, MERGE_READ_CACHE_BYTES, PAGED_SEGMENT_ENVELOPE_VERSION, PAYLOAD_PREFIX_BYTES, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, POSTING_WRITE_POSITIONS, POSTING_WRITE_ROWS, READS_AHEAD_AT_ONCE, READ_AHEAD_CHUNK_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_TARGET_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, REWRITE_SPILL_BYTES, SECTION_ALIGNMENT_BYTES, SECTION_WRITE_BUFFER_BYTES, SEGMENT_PAGE_BYTES, SMALLEST_BLOCK_POSTINGS, STABLE_HEAD_ATTEMPTS, TOMBSTONES_BETWEEN_YIELDS, WRITER_YIELD_INTERVAL_TOKENS, adapterCheckpointStore, cachedFileBytes, checkEverySectionPage, closePagedFile, createReadCache, createSectionReader, decodePagedSegmentHead, decodeSections, directoryCheckpointStore, encodePagedSegmentHead, encodeSegmentManifest, failNativePage, loadFailed, openPagedFile, readCacheOfThisThread, readFileRange, readFileRangeLater, readListedFile, readWholeSectionChecked, requireListedChecksum, requireListedSize, requireNativePage, resumePagedFile, sectionNamed, suspendPagedFile };
|