@delali/narsil 0.3.1-next.99 → 0.4.1-next.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +30 -11
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +404 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
package/NOTICE
CHANGED
|
@@ -52,3 +52,48 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
|
|
52
52
|
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
53
53
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
54
54
|
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
55
|
+
|
|
56
|
+
--------------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
This product includes software derived from fdlibm
|
|
59
|
+
(http://www.netlib.org/fdlibm/), through the copy in the V8 project's
|
|
60
|
+
src/base/ieee754.cc.
|
|
61
|
+
|
|
62
|
+
The functions under packages/ts/src/math/ port its logarithm and trigonometric
|
|
63
|
+
functions to TypeScript, so that the engine computes the same results as the
|
|
64
|
+
native core on every platform. The two notices below cover the port.
|
|
65
|
+
|
|
66
|
+
Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
|
|
67
|
+
|
|
68
|
+
Developed at SunSoft, a Sun Microsystems, Inc. business.
|
|
69
|
+
Permission to use, copy, modify, and distribute this
|
|
70
|
+
software is freely granted, provided that this notice
|
|
71
|
+
is preserved.
|
|
72
|
+
|
|
73
|
+
Copyright 2016 the V8 project authors. All rights reserved.
|
|
74
|
+
|
|
75
|
+
Redistribution and use in source and binary forms, with or without
|
|
76
|
+
modification, are permitted provided that the following conditions are
|
|
77
|
+
met:
|
|
78
|
+
|
|
79
|
+
* Redistributions of source code must retain the above copyright
|
|
80
|
+
notice, this list of conditions and the following disclaimer.
|
|
81
|
+
* Redistributions in binary form must reproduce the above
|
|
82
|
+
copyright notice, this list of conditions and the following
|
|
83
|
+
disclaimer in the documentation and/or other materials provided
|
|
84
|
+
with the distribution.
|
|
85
|
+
* Neither the name of Google Inc. nor the names of its
|
|
86
|
+
contributors may be used to endorse or promote products derived
|
|
87
|
+
from this software without specific prior written permission.
|
|
88
|
+
|
|
89
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
90
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
91
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
92
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
93
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
94
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
95
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
96
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
97
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
98
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
99
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ The search engine that scales with your data.
|
|
|
12
12
|
|
|
13
13
|
Narsil is an open source search engine with full-text, vector, hybrid, and geosearch on one typed index. You can embed Narsil in your application process, where the engine searches without a network hop, or deploy it as a standalone search server with a REST API, a write-ahead log, and bulk NDJSON ingest. Both forms contain the same engine, which stores indexes in one cross-language binary format (`.nrsl`), so you can load an index from one form into the other.
|
|
14
14
|
|
|
15
|
-
The engine partitions large indexes across workers and merges the results of the partitions into a single ranked list. This TypeScript package is the reference implementation of the `.nrsl` format,
|
|
15
|
+
The engine partitions large indexes across workers and merges the results of the partitions into a single ranked list. This TypeScript package is the reference implementation of the `.nrsl` format, and [a Rust crate and a Go module](#other-languages) hold the same engine for programs in those languages.
|
|
16
16
|
|
|
17
17
|
<!-- BENCH:headline START -->
|
|
18
18
|
On BEIR SciFact, Narsil's BM25 ranking scores 0.681 nDCG@10, which is within 0.002 of the Anserini reference of 0.679. Elasticsearch scores 0.679 and OpenSearch scores 0.679 on the same queries. On SciFact, Narsil answers 4,200 keyword queries a second at its peak, and OpenSearch follows with 3,027. On DBpedia entities 1M with every engine held at 0.99 recall, Narsil answers 670 vector queries a second at its peak, while OpenSearch leads with 869. With each engine's recommended production settings on the same set, Narsil answers 1,154 vector queries a second at its peak, and OpenSearch follows with 1,111. Narsil has the highest peak throughput in 16 of the 20 keyword, vector, and hybrid comparisons in the newest recorded run: 8 of 12 with every engine on equal settings, and all 8 with each engine's recommended production settings. The harness recorded these figures on 2026-09-28 in run `20260928T002255Z`. GCP c3-standard-8 hardware in us-central1-a, us-east1-b, and us-west1-a hosted the engines. You'll find every dataset, every engine's settings, and the full method in [`BENCHMARKS.md`](https://github.com/assetcorp/narsil/blob/main/BENCHMARKS.md).
|
|
@@ -33,23 +33,24 @@ Try it in your browser at [narsil.sondelali.com/demo](https://narsil.sondelali.c
|
|
|
33
33
|
- [Distribution](#distribution)
|
|
34
34
|
- [Search quality](#search-quality)
|
|
35
35
|
- [Runtime support](#runtime-support)
|
|
36
|
+
- [Other languages](#other-languages)
|
|
36
37
|
- [License](#license)
|
|
37
38
|
|
|
38
39
|
## Install
|
|
39
40
|
|
|
40
|
-
```bash
|
|
41
|
+
```bash skip
|
|
41
42
|
pnpm add @delali/narsil
|
|
42
43
|
```
|
|
43
44
|
|
|
44
45
|
Narsil works in Node.js 22 or newer, and in Bun, Deno, and browsers. The [Runtime support](#runtime-support) table covers each runtime.
|
|
45
46
|
|
|
46
|
-
On an arm64 or x64 machine under macOS, Linux, or Windows, the package manager also installs Narsil's native search core as an optional dependency, so the engine
|
|
47
|
+
On an arm64 or x64 machine under macOS, Linux, or Windows, the package manager also installs Narsil's native search core in C as an optional dependency, so the engine analyses text and answers keyword and vector searches through it. If you omit optional dependencies, Narsil does that work through WebAssembly and TypeScript, with the same results, as it always does in a browser.
|
|
47
48
|
|
|
48
49
|
## Add Narsil with a coding agent
|
|
49
50
|
|
|
50
51
|
The [Narsil skill](https://github.com/assetcorp/narsil/tree/main/skills/narsil) tells a coding agent whether to embed Narsil or serve it, which packages to install alongside it, and which server settings keep a public address safe. Install the skill into the coding agents on your machine with the skills CLI:
|
|
51
52
|
|
|
52
|
-
```bash
|
|
53
|
+
```bash skip
|
|
53
54
|
npx skills add assetcorp/narsil
|
|
54
55
|
```
|
|
55
56
|
|
|
@@ -57,7 +58,7 @@ The skill has the agent read the Narsil version that your project installs, and
|
|
|
57
58
|
|
|
58
59
|
## Quick start
|
|
59
60
|
|
|
60
|
-
```ts
|
|
61
|
+
```ts run
|
|
61
62
|
import { createNarsil } from '@delali/narsil'
|
|
62
63
|
|
|
63
64
|
const narsil = await createNarsil()
|
|
@@ -102,15 +103,15 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
102
103
|
|
|
103
104
|
**Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md). On a field that you declare for [pattern search](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md#pattern-tests), a filter can find text anywhere inside a value with `contains`, test the whole value against a wildcard pattern, look for a match of a regular expression, and accept a typo or two in an `eq` test. With [partial-word search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md#matching-inside-words), a query word also matches inside longer words, so a search for `proof` finds `waterproof`.
|
|
104
105
|
|
|
105
|
-
**Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter
|
|
106
|
+
**Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter that you write. A query can [rerank](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) its top hits with a cross-encoder, through a Cohere-compatible rerank service, a local Transformers.js model, or an adapter that you write.
|
|
106
107
|
|
|
107
108
|
**Geosearch.** [Geo filters](https://github.com/assetcorp/narsil/blob/main/docs/geosearch.md) match documents by radius, using Haversine or Vincenty distance, or by polygon containment. You can combine a geo filter with every other query feature.
|
|
108
109
|
|
|
109
|
-
**Storage.** [Persistence adapters](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) store indexes on the filesystem, in IndexedDB, in memory, or in a backend
|
|
110
|
+
**Storage.** [Persistence adapters](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) store indexes on the filesystem, in IndexedDB, in memory, or in a backend that you write. With durability on, the engine keeps a write-ahead log with periodic checkpoints and recovers automatically after a crash. You can stream a whole index into a portable snapshot archive, or export its documents as JSON Lines to move the index across releases. With [device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md), a phone, a laptop, or a browser tab keeps a copy of one index from a server, which it searches and writes while it is offline.
|
|
110
111
|
|
|
111
112
|
**Scale.** The engine routes each document to a [partition](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) by a deterministic hash. When you call `rebalance()`, the engine reshapes the partitions online and buffers incoming writes in a write-ahead queue. Once an index holds 1,000 documents, the engine serves keyword queries from worker copies on half its worker threads, while the HTTP server receives requests on the threads that hold those copies.
|
|
112
113
|
|
|
113
|
-
**Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover
|
|
114
|
+
**Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover the languages that the guide lists, including Swahili, Yoruba, Hausa, and other African languages, and each module is a separate entry point.
|
|
114
115
|
|
|
115
116
|
## Documentation
|
|
116
117
|
|
|
@@ -127,10 +128,16 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
127
128
|
| [Reranking](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) | Reordering the top hits with a cross-encoder, the bundled rerank adapters, and custom ones |
|
|
128
129
|
| [Persistence and durability](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) | Storage backends, the write-ahead log, checkpoints, recovery, snapshots, document exports, and indexes from an earlier release |
|
|
129
130
|
| [Partitions and workers](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) | Partition routing, online rebalancing, worker copies, and multi-instance invalidation |
|
|
130
|
-
| [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) |
|
|
131
|
+
| [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) | Every language module, analysis revisions and rebuilds, and named tokenizers and stop words |
|
|
131
132
|
| [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) | Wrapping an engine in a REST API, every route it serves, and long-running tasks |
|
|
132
133
|
| [Cluster mode](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) | Multi-node indexes: nodes and roles, replication, routed writes, distributed searches and reads, and the calls that fail in a cluster |
|
|
133
134
|
| [Client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) | Reaching a server from a browser or Node, following a task, and the codes it raises |
|
|
135
|
+
| [Python client](https://github.com/assetcorp/narsil/blob/main/docs/python-client.md) | Reaching a server or a cluster from Python, in blocking and async forms, and the failover rules |
|
|
136
|
+
| [Go client](https://github.com/assetcorp/narsil/blob/main/docs/go-client.md) | Reaching a server or a cluster from Go, streaming snapshots, and the failover rules |
|
|
137
|
+
| [React Native](https://github.com/assetcorp/narsil/blob/main/docs/react-native.md) | The engine inside an iOS or Android app, where the index lives, snapshots, memory, and errors |
|
|
138
|
+
| [Device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md) | A copy of one index on a device that searches and writes offline, the sync routes, conflicts, and the snapshot |
|
|
139
|
+
| [Change capture](https://github.com/assetcorp/narsil/blob/main/docs/change-capture.md) | Following every insert, update, and removal on an index from a backend service, the feed, filters, and resyncs |
|
|
140
|
+
| [Popular query suggestions](https://github.com/assetcorp/narsil/blob/main/docs/query-suggestions.md) | Whole searches that people ran and clicked, counted per person, built into a suggestions index, and the privacy trade-offs |
|
|
134
141
|
| [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over a client or an engine in the page, one shared request per key, and loading a corpus from a component |
|
|
135
142
|
| [Observability](https://github.com/assetcorp/narsil/blob/main/docs/observability.md) | Plugin hooks, engine events, and memory reporting |
|
|
136
143
|
| [Errors](https://github.com/assetcorp/narsil/blob/main/docs/errors.md) | The codes you handle most often, and the three sets that `NarsilError.code` covers |
|
|
@@ -147,11 +154,11 @@ The [specification](https://github.com/assetcorp/narsil/tree/main/packages/spec)
|
|
|
147
154
|
|
|
148
155
|
## Distribution
|
|
149
156
|
|
|
150
|
-
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node can create, drop, clear, write, update, search, list, count, and suggest across every partition. The [cluster dashboard example](https://github.com/assetcorp/narsil/tree/main/packages/ts/examples/cluster-dashboard) starts three nodes against etcd and shows the state of each partition while you cut the network links.
|
|
157
|
+
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node can create, drop, clear, write, update, search, list, count, and suggest across every partition, and it can change the replication factor and the partition count of an index. A node with a durability directory keeps its id there across restarts, and the etcd coordinator connects over TLS with a user and a password. A node that the HTTP server fronts reports the health of the cluster and the node's Prometheus metrics. The [cluster dashboard example](https://github.com/assetcorp/narsil/tree/main/packages/ts/examples/cluster-dashboard) starts three nodes against etcd and shows the state of each partition while you cut the network links. The [cluster guide](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) covers the API, including the packages that the etcd coordinator and the gRPC transport each need. For the contract that every implementation must uphold, read [`packages/spec/distribution`](https://github.com/assetcorp/narsil/tree/main/packages/spec/distribution).
|
|
151
158
|
|
|
152
159
|
## Search quality
|
|
153
160
|
|
|
154
|
-
We measure ranking quality on the [BEIR](https://github.com/beir-cellar/beir) SciFact corpus, which holds 5,183 documents and 300 queries with human relevance judgements. We compare Narsil with Orama and MiniSearch in one process. All three use the same 33 Lucene English stop words and default BM25 parameters, although each one stems English
|
|
161
|
+
We measure ranking quality on the [BEIR](https://github.com/beir-cellar/beir) SciFact corpus, which holds 5,183 documents and 300 queries with human relevance judgements. We compare Narsil with Orama and MiniSearch in one process. All three use the same 33 Lucene English stop words and default BM25 parameters, although each one stems English differently. Narsil has the top nDCG@10 of the three.
|
|
155
162
|
|
|
156
163
|
**What these metrics mean:**
|
|
157
164
|
|
|
@@ -177,6 +184,18 @@ Narsil also works as a search server. On the BEIR datasets, we compare the serve
|
|
|
177
184
|
|
|
178
185
|
The [browser example](https://github.com/assetcorp/narsil/blob/main/packages/ts/examples/browser/README.md) shows an embedded engine with IndexedDB persistence, while the [server app example](https://github.com/assetcorp/narsil/blob/main/packages/ts/examples/server-app/README.md) shows the same interface over the client SDK and the React hooks.
|
|
179
186
|
|
|
187
|
+
## Other languages
|
|
188
|
+
|
|
189
|
+
| Package | What it does |
|
|
190
|
+
| --- | --- |
|
|
191
|
+
| [`delali-narsil`](https://github.com/assetcorp/narsil/tree/main/packages/rust) | The Rust crate creates, changes, and searches an index inside a Rust program. |
|
|
192
|
+
| [`github.com/assetcorp/narsil/packages/go/engine`](https://github.com/assetcorp/narsil/tree/main/packages/go/engine) | The Go engine creates, changes, and searches an index inside a Go program. |
|
|
193
|
+
| [`github.com/assetcorp/narsil/packages/go`](https://github.com/assetcorp/narsil/tree/main/packages/go) | The Go client calls a Narsil server, or every node of a cluster, from Go. |
|
|
194
|
+
| [`narsil-client`](https://github.com/assetcorp/narsil/tree/main/packages/python) | The Python client calls a Narsil server, or every node of a cluster, from Python. |
|
|
195
|
+
| [`@delali/narsil-react-native`](https://github.com/assetcorp/narsil/tree/main/packages/react-native) | The driver puts the engine inside an iOS or Android app. |
|
|
196
|
+
|
|
197
|
+
The Rust crate and the Go engine save every file in the same layouts as this package, so this package can open an index that either one saves, and either one can open an index that this package saves. Only this package holds the HTTP server and cluster mode.
|
|
198
|
+
|
|
180
199
|
## License
|
|
181
200
|
|
|
182
201
|
Apache-2.0
|
|
@@ -1,6 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { requireStreamLength, countedChunks } from '../chunk-GR5GMFJJ.mjs';
|
|
2
|
+
import '../chunk-4PY4JYMY.mjs';
|
|
3
|
+
import { OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../chunk-ZKOQR7EA.mjs';
|
|
2
5
|
|
|
3
6
|
// src/persistence/filesystem.ts
|
|
7
|
+
function unfinishedSaveName(basename) {
|
|
8
|
+
return `${basename}~tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
9
|
+
}
|
|
10
|
+
async function* readChunks(handle) {
|
|
11
|
+
try {
|
|
12
|
+
let remaining = (await handle.stat()).size;
|
|
13
|
+
while (remaining > 0) {
|
|
14
|
+
const buffer = new Uint8Array(Math.min(remaining, OBJECT_STORE_PART_BYTES));
|
|
15
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, null);
|
|
16
|
+
if (bytesRead === 0) return;
|
|
17
|
+
remaining -= bytesRead;
|
|
18
|
+
yield bytesRead === buffer.byteLength ? buffer : buffer.subarray(0, bytesRead);
|
|
19
|
+
}
|
|
20
|
+
} finally {
|
|
21
|
+
await handle.close().catch(() => void 0);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
4
24
|
var cachedFs = null;
|
|
5
25
|
var cachedPath = null;
|
|
6
26
|
async function getFs() {
|
|
@@ -49,6 +69,25 @@ async function fsyncDirectory(dirPath, fs) {
|
|
|
49
69
|
}
|
|
50
70
|
}
|
|
51
71
|
}
|
|
72
|
+
async function writeSyncedFile(fs, path, data, key) {
|
|
73
|
+
let handle = null;
|
|
74
|
+
try {
|
|
75
|
+
handle = await fs.open(path, "w");
|
|
76
|
+
await handle.write(data);
|
|
77
|
+
try {
|
|
78
|
+
await handle.sync();
|
|
79
|
+
} catch (err) {
|
|
80
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_FSYNC_FAILED, `fsync failed for "${key}"`, {
|
|
81
|
+
key,
|
|
82
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
await handle.close();
|
|
86
|
+
handle = null;
|
|
87
|
+
} finally {
|
|
88
|
+
if (handle !== null) await handle.close().catch(() => void 0);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
52
91
|
async function listDirectoryRecursive(dir, baseDir, pathMod, fs) {
|
|
53
92
|
const results = [];
|
|
54
93
|
let entries;
|
|
@@ -88,31 +127,80 @@ function createFilesystemPersistence(config) {
|
|
|
88
127
|
const pathMod = await getPath();
|
|
89
128
|
const dir = pathMod.dirname(resolvedPath);
|
|
90
129
|
await fs.mkdir(dir, { recursive: true });
|
|
91
|
-
const
|
|
92
|
-
const tempPath = pathMod.join(dir, tempName);
|
|
93
|
-
let handle = null;
|
|
130
|
+
const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
|
|
94
131
|
try {
|
|
95
|
-
|
|
96
|
-
await
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
132
|
+
await writeSyncedFile(fs, tempPath, data, key);
|
|
133
|
+
await fs.rename(tempPath, resolvedPath);
|
|
134
|
+
await fsyncDirectory(dir, fs);
|
|
135
|
+
} catch (err) {
|
|
136
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
async saveIfAbsent(key, data) {
|
|
141
|
+
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
142
|
+
const fs = await getFs();
|
|
143
|
+
const pathMod = await getPath();
|
|
144
|
+
const dir = pathMod.dirname(resolvedPath);
|
|
145
|
+
const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
|
|
146
|
+
let stored = false;
|
|
147
|
+
try {
|
|
148
|
+
await fs.mkdir(dir, { recursive: true });
|
|
149
|
+
await writeSyncedFile(fs, tempPath, data, key);
|
|
150
|
+
await fs.link(tempPath, resolvedPath);
|
|
151
|
+
stored = true;
|
|
152
|
+
} catch (err) {
|
|
153
|
+
if (err.code !== "EEXIST") {
|
|
154
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
155
|
+
if (err instanceof NarsilError) throw err;
|
|
156
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The adapter could not save "${key}"`, {
|
|
101
157
|
key,
|
|
102
158
|
cause: err instanceof Error ? err.message : String(err)
|
|
103
159
|
});
|
|
104
160
|
}
|
|
161
|
+
}
|
|
162
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
163
|
+
await fsyncDirectory(dir, fs);
|
|
164
|
+
return stored;
|
|
165
|
+
},
|
|
166
|
+
async saveStream(key, data, length) {
|
|
167
|
+
requireStreamLength(key, length);
|
|
168
|
+
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
169
|
+
const fs = await getFs();
|
|
170
|
+
const pathMod = await getPath();
|
|
171
|
+
const dir = pathMod.dirname(resolvedPath);
|
|
172
|
+
await fs.mkdir(dir, { recursive: true });
|
|
173
|
+
const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
|
|
174
|
+
let handle = null;
|
|
175
|
+
try {
|
|
176
|
+
handle = await fs.open(tempPath, "w");
|
|
177
|
+
for await (const chunk of countedChunks(key, data, length)) await handle.write(chunk);
|
|
178
|
+
await handle.sync();
|
|
105
179
|
await handle.close();
|
|
106
180
|
handle = null;
|
|
107
181
|
await fs.rename(tempPath, resolvedPath);
|
|
108
182
|
await fsyncDirectory(dir, fs);
|
|
109
183
|
} catch (err) {
|
|
110
|
-
if (handle !== null)
|
|
111
|
-
await handle.close().catch(() => void 0);
|
|
112
|
-
}
|
|
184
|
+
if (handle !== null) await handle.close().catch(() => void 0);
|
|
113
185
|
await fs.unlink(tempPath).catch(() => void 0);
|
|
186
|
+
if (err instanceof NarsilError) throw err;
|
|
187
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The adapter could not save "${key}"`, {
|
|
188
|
+
key,
|
|
189
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
async loadStream(key) {
|
|
194
|
+
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
195
|
+
const fs = await getFs();
|
|
196
|
+
let handle;
|
|
197
|
+
try {
|
|
198
|
+
handle = await fs.open(resolvedPath, "r");
|
|
199
|
+
} catch (err) {
|
|
200
|
+
if (err.code === "ENOENT") return null;
|
|
114
201
|
throw err;
|
|
115
202
|
}
|
|
203
|
+
return readChunks(handle);
|
|
116
204
|
},
|
|
117
205
|
async load(key) {
|
|
118
206
|
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The service account key that {@link createGcsPersistence} signs its requests with.
|
|
5
|
+
*
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface GcsCredentials {
|
|
9
|
+
/** The service account's email address. */
|
|
10
|
+
client_email: string;
|
|
11
|
+
/** The service account's private key in PEM form. */
|
|
12
|
+
private_key: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Where {@link createGcsPersistence} stores its objects, and how it reaches Google Cloud Storage.
|
|
16
|
+
*
|
|
17
|
+
* @public
|
|
18
|
+
*/
|
|
19
|
+
interface GcsPersistenceConfig {
|
|
20
|
+
/** The adapter writes every object to this bucket. */
|
|
21
|
+
bucket: string;
|
|
22
|
+
/** The project that owns the bucket, which the SDK otherwise reads from the environment. */
|
|
23
|
+
projectId?: string;
|
|
24
|
+
/** The path of a service account key file, which the SDK otherwise finds through its default credentials. */
|
|
25
|
+
keyFilename?: string;
|
|
26
|
+
/** A service account key held in memory, as an alternative to `keyFilename`. */
|
|
27
|
+
credentials?: GcsCredentials;
|
|
28
|
+
/** The address of a service that speaks the Cloud Storage JSON API, such as an emulator. */
|
|
29
|
+
apiEndpoint?: string;
|
|
30
|
+
/** The adapter stores every key under this prefix inside the bucket, so several stores can share one bucket. */
|
|
31
|
+
keyPrefix?: string;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Builds a persistence adapter that stores each key as one object in a Google Cloud Storage bucket.
|
|
35
|
+
*
|
|
36
|
+
* Cluster backups use it as the store outside the cluster. The adapter loads `@google-cloud/storage`, an optional
|
|
37
|
+
* peer dependency, when you call this function, checks each download against the checksum that Cloud Storage keeps,
|
|
38
|
+
* and refuses to load an object larger than one stored file can be.
|
|
39
|
+
*
|
|
40
|
+
* @param config - The bucket, the project, and the credentials.
|
|
41
|
+
* @returns An adapter you pass wherever Narsil takes a {@link PersistenceAdapter}.
|
|
42
|
+
* @throws A `NarsilError` with `PERSISTENCE_DEPENDENCY_MISSING` and the install command when the SDK is missing, and
|
|
43
|
+
* with `CONFIG_INVALID` when the bucket name is empty.
|
|
44
|
+
*
|
|
45
|
+
* @public
|
|
46
|
+
*/
|
|
47
|
+
declare function createGcsPersistence(config: GcsPersistenceConfig): Promise<PersistenceAdapter>;
|
|
48
|
+
|
|
49
|
+
export { type GcsCredentials, type GcsPersistenceConfig, createGcsPersistence };
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { requireBucket, objectKeyPrefix, objectStoreFailure, requireObjectKey, requireStreamLength, countedChunks, readWhole, refuseOversizedObject, joinChunks } from '../chunk-GR5GMFJJ.mjs';
|
|
2
|
+
import { MAX_FILE_BYTES } from '../chunk-4PY4JYMY.mjs';
|
|
3
|
+
import { OBJECT_STORE_LIST_PAGE_KEYS, OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
|
|
4
|
+
import { ErrorCodes, NarsilError } from '../chunk-ZKOQR7EA.mjs';
|
|
5
|
+
import { Readable } from 'stream';
|
|
6
|
+
import { pipeline } from 'stream/promises';
|
|
7
|
+
|
|
8
|
+
var GCS_NOT_FOUND_STATUS = 404;
|
|
9
|
+
var GCS_PRECONDITION_FAILED_STATUS = 412;
|
|
10
|
+
var GCS_CONTENT_TYPE = "application/octet-stream";
|
|
11
|
+
async function loadGcsModule() {
|
|
12
|
+
try {
|
|
13
|
+
return await import('@google-cloud/storage');
|
|
14
|
+
} catch (err) {
|
|
15
|
+
throw new NarsilError(
|
|
16
|
+
ErrorCodes.PERSISTENCE_DEPENDENCY_MISSING,
|
|
17
|
+
"The `@google-cloud/storage` package is not installed. Install it with `pnpm add @google-cloud/storage` (or the npm or yarn equivalent) to use createGcsPersistence.",
|
|
18
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function statusOf(error) {
|
|
23
|
+
return typeof error === "object" && error !== null ? error.code : void 0;
|
|
24
|
+
}
|
|
25
|
+
function isNotFound(error) {
|
|
26
|
+
return statusOf(error) === GCS_NOT_FOUND_STATUS;
|
|
27
|
+
}
|
|
28
|
+
function isPreconditionFailed(error) {
|
|
29
|
+
return statusOf(error) === GCS_PRECONDITION_FAILED_STATUS;
|
|
30
|
+
}
|
|
31
|
+
function readObject(file, key) {
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
const chunks = [];
|
|
34
|
+
let total = 0;
|
|
35
|
+
let settled = false;
|
|
36
|
+
const stream = file.createReadStream();
|
|
37
|
+
stream.on("data", (chunk) => {
|
|
38
|
+
total += chunk.byteLength;
|
|
39
|
+
if (total > MAX_FILE_BYTES) {
|
|
40
|
+
stream.destroy();
|
|
41
|
+
if (!settled) {
|
|
42
|
+
settled = true;
|
|
43
|
+
try {
|
|
44
|
+
refuseOversizedObject(key, total);
|
|
45
|
+
} catch (error) {
|
|
46
|
+
reject(error);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
chunks.push(chunk);
|
|
52
|
+
});
|
|
53
|
+
stream.on("error", (error) => {
|
|
54
|
+
if (settled) return;
|
|
55
|
+
settled = true;
|
|
56
|
+
if (isNotFound(error)) resolve(null);
|
|
57
|
+
else reject(objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error));
|
|
58
|
+
});
|
|
59
|
+
stream.on("end", () => {
|
|
60
|
+
if (settled) return;
|
|
61
|
+
settled = true;
|
|
62
|
+
resolve(joinChunks(chunks, total));
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async function* remainingChunks(key, first, rest, stream) {
|
|
67
|
+
let total = first.byteLength;
|
|
68
|
+
try {
|
|
69
|
+
refuseOversizedObject(key, total);
|
|
70
|
+
yield first;
|
|
71
|
+
for (; ; ) {
|
|
72
|
+
const step = await rest.next();
|
|
73
|
+
if (step.done === true) return;
|
|
74
|
+
total += step.value.byteLength;
|
|
75
|
+
refuseOversizedObject(key, total);
|
|
76
|
+
yield step.value;
|
|
77
|
+
}
|
|
78
|
+
} catch (error) {
|
|
79
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
80
|
+
} finally {
|
|
81
|
+
stream.destroy();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async function* chunksOf(chunks) {
|
|
85
|
+
yield* chunks;
|
|
86
|
+
}
|
|
87
|
+
async function openObjectStream(file, key) {
|
|
88
|
+
const stream = file.createReadStream();
|
|
89
|
+
const chunks = stream[Symbol.asyncIterator]();
|
|
90
|
+
let first;
|
|
91
|
+
try {
|
|
92
|
+
first = await chunks.next();
|
|
93
|
+
} catch (error) {
|
|
94
|
+
stream.destroy();
|
|
95
|
+
if (isNotFound(error)) return null;
|
|
96
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
97
|
+
}
|
|
98
|
+
if (first.done === true) {
|
|
99
|
+
stream.destroy();
|
|
100
|
+
return chunksOf([]);
|
|
101
|
+
}
|
|
102
|
+
return remainingChunks(key, first.value, chunks, stream);
|
|
103
|
+
}
|
|
104
|
+
async function createGcsPersistence(config) {
|
|
105
|
+
const bucketName = requireBucket(config.bucket, "Google Cloud Storage");
|
|
106
|
+
const root = objectKeyPrefix(config.keyPrefix, "Google Cloud Storage");
|
|
107
|
+
const sdk = await loadGcsModule();
|
|
108
|
+
const storage = new sdk.Storage({
|
|
109
|
+
...config.projectId === void 0 ? {} : { projectId: config.projectId },
|
|
110
|
+
...config.keyFilename === void 0 ? {} : { keyFilename: config.keyFilename },
|
|
111
|
+
...config.credentials === void 0 ? {} : { credentials: config.credentials },
|
|
112
|
+
...config.apiEndpoint === void 0 ? {} : { apiEndpoint: config.apiEndpoint },
|
|
113
|
+
retryOptions: { idempotencyStrategy: sdk.IdempotencyStrategy.RetryAlways }
|
|
114
|
+
});
|
|
115
|
+
const bucket = storage.bucket(bucketName);
|
|
116
|
+
return {
|
|
117
|
+
async save(key, data) {
|
|
118
|
+
requireObjectKey(key);
|
|
119
|
+
try {
|
|
120
|
+
await bucket.file(`${root}${key}`).save(data, { resumable: false, contentType: GCS_CONTENT_TYPE });
|
|
121
|
+
} catch (error) {
|
|
122
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
async saveIfAbsent(key, data) {
|
|
126
|
+
requireObjectKey(key);
|
|
127
|
+
try {
|
|
128
|
+
await bucket.file(`${root}${key}`).save(data, {
|
|
129
|
+
resumable: false,
|
|
130
|
+
contentType: GCS_CONTENT_TYPE,
|
|
131
|
+
preconditionOpts: { ifGenerationMatch: 0 }
|
|
132
|
+
});
|
|
133
|
+
return true;
|
|
134
|
+
} catch (error) {
|
|
135
|
+
if (isPreconditionFailed(error)) return false;
|
|
136
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
async load(key) {
|
|
140
|
+
requireObjectKey(key);
|
|
141
|
+
return readObject(bucket.file(`${root}${key}`), key);
|
|
142
|
+
},
|
|
143
|
+
async saveStream(key, data, length) {
|
|
144
|
+
requireObjectKey(key);
|
|
145
|
+
try {
|
|
146
|
+
requireStreamLength(key, length);
|
|
147
|
+
const counted = countedChunks(key, data, length);
|
|
148
|
+
const file = bucket.file(`${root}${key}`);
|
|
149
|
+
if (length <= OBJECT_STORE_PART_BYTES) {
|
|
150
|
+
await file.save(await readWhole(counted, length), { resumable: false, contentType: GCS_CONTENT_TYPE });
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
await pipeline(
|
|
154
|
+
Readable.from(counted, { objectMode: false }),
|
|
155
|
+
file.createWriteStream({
|
|
156
|
+
resumable: true,
|
|
157
|
+
chunkSize: OBJECT_STORE_PART_BYTES,
|
|
158
|
+
contentType: GCS_CONTENT_TYPE
|
|
159
|
+
})
|
|
160
|
+
);
|
|
161
|
+
} catch (error) {
|
|
162
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
163
|
+
}
|
|
164
|
+
},
|
|
165
|
+
async loadStream(key) {
|
|
166
|
+
requireObjectKey(key);
|
|
167
|
+
return openObjectStream(bucket.file(`${root}${key}`), key);
|
|
168
|
+
},
|
|
169
|
+
async delete(key) {
|
|
170
|
+
requireObjectKey(key);
|
|
171
|
+
const file = bucket.file(`${root}${key}`);
|
|
172
|
+
try {
|
|
173
|
+
const [metadata] = await file.getMetadata();
|
|
174
|
+
const generation = metadata.generation;
|
|
175
|
+
if (generation === void 0) {
|
|
176
|
+
throw new NarsilError(
|
|
177
|
+
ErrorCodes.PERSISTENCE_DELETE_FAILED,
|
|
178
|
+
`Cloud Storage reported no generation for "${key}", so the adapter deletes nothing`,
|
|
179
|
+
{ key }
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
await file.delete({ ignoreNotFound: true, ifGenerationMatch: generation });
|
|
183
|
+
} catch (error) {
|
|
184
|
+
if (isNotFound(error) || isPreconditionFailed(error)) return;
|
|
185
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key, error);
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
async list(prefix) {
|
|
189
|
+
const keys = [];
|
|
190
|
+
let pageToken;
|
|
191
|
+
try {
|
|
192
|
+
do {
|
|
193
|
+
const [files, next] = await bucket.getFiles({
|
|
194
|
+
prefix: `${root}${prefix}`,
|
|
195
|
+
autoPaginate: false,
|
|
196
|
+
maxResults: OBJECT_STORE_LIST_PAGE_KEYS,
|
|
197
|
+
...pageToken === void 0 ? {} : { pageToken }
|
|
198
|
+
});
|
|
199
|
+
for (const file of files) {
|
|
200
|
+
if (file.name.startsWith(root)) keys.push(file.name.slice(root.length));
|
|
201
|
+
}
|
|
202
|
+
const token = next !== null && typeof next === "object" && "pageToken" in next ? next.pageToken : void 0;
|
|
203
|
+
pageToken = typeof token === "string" && token.length > 0 ? token : void 0;
|
|
204
|
+
} while (pageToken !== void 0);
|
|
205
|
+
} catch (error) {
|
|
206
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "list the keys under", prefix, error);
|
|
207
|
+
}
|
|
208
|
+
return keys;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export { createGcsPersistence };
|
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
// src/persistence/constants.ts
|
|
4
|
-
var DEFAULT_INDEXEDDB_NAME = "narsil";
|
|
5
|
-
var DEFAULT_INDEXEDDB_STORE_NAME = "partitions";
|
|
6
|
-
var MAX_PERSISTENCE_KEY_LENGTH = 1024;
|
|
7
|
-
var INDEXEDDB_BLOCKED_TIMEOUT_MS = 5e3;
|
|
1
|
+
import { DEFAULT_INDEXEDDB_NAME, DEFAULT_INDEXEDDB_STORE_NAME, INDEXEDDB_BLOCKED_TIMEOUT_MS, MAX_PERSISTENCE_KEY_LENGTH } from '../chunk-K5HSUPYS.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from '../chunk-ZKOQR7EA.mjs';
|
|
8
3
|
|
|
9
4
|
// src/persistence/indexeddb.ts
|
|
10
5
|
function validateKey(key) {
|
package/dist/adapters/memory.mjs
CHANGED
|
@@ -5,6 +5,11 @@ function createMemoryPersistence() {
|
|
|
5
5
|
async save(key, data) {
|
|
6
6
|
store.set(key, new Uint8Array(data));
|
|
7
7
|
},
|
|
8
|
+
async saveIfAbsent(key, data) {
|
|
9
|
+
if (store.has(key)) return false;
|
|
10
|
+
store.set(key, new Uint8Array(data));
|
|
11
|
+
return true;
|
|
12
|
+
},
|
|
8
13
|
async load(key) {
|
|
9
14
|
const stored = store.get(key);
|
|
10
15
|
if (stored === void 0) {
|