@delali/narsil 0.3.1-next.144 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
- package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/chunk-3TVDTACC.mjs +104 -0
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
- package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/chunk-6BWLDLS3.mjs +58 -0
- package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
- package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/chunk-B7XIOAMK.mjs +100 -0
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
- package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
- package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
- package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
- package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
- package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
- package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/chunk-SOUZMPME.mjs +1635 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
- package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
- package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
- package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
- package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/chunk-YKEHQUSZ.mjs +248 -0
- package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +46 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
- package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16116 -14537
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +542 -8
- package/dist/index.mjs +29 -32
- package/dist/index.react-native.mjs +19 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
- package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
- package/dist/module-V5K4ZP23.mjs +45 -0
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +11 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +159 -38
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
- package/openapi.json +349 -2
- package/package.json +108 -8
- package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
- package/dist/module-IYPLWHGC.mjs +0 -48
- package/dist/thread-KGNAIKPD.mjs +0 -33
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -1,30 +1,260 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-
|
|
2
|
-
import {
|
|
1
|
+
import { DEFAULT_BLOCK_WRITES_ABOVE, DEFAULT_RESUME_WRITES_BELOW, validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, ENGINE_MEMORY_FIGURES_MAX_AGE_MS, storedBM25Params, readEngineMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBatchList, validateDocId } from './chunk-6F6USYII.mjs';
|
|
2
|
+
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, generateId, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-O4KUNDVF.mjs';
|
|
3
|
+
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, readHeapStatistics, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, encodeSegmentState } from './chunk-3HV7M5OJ.mjs';
|
|
4
|
+
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
3
5
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
+
import { detectRuntime, resolveWorkerEntry, missingWorkerEntry, workerResourceLimits } from './chunk-6BWLDLS3.mjs';
|
|
7
|
+
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-XIHUI4KB.mjs';
|
|
8
|
+
import { countEngineThread } from './chunk-Q2UL2TPC.mjs';
|
|
9
|
+
import { splitWorkerBudget, resolveWorkerCount, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-3MXV3VHR.mjs';
|
|
6
10
|
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { generateId } from './chunk-N37VMC62.mjs';
|
|
10
|
-
import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
|
|
11
|
-
import { countEngineThread } from './chunk-EBIHRL7B.mjs';
|
|
12
|
-
import { requireSearchBackendSettings } from './chunk-74XB2SXI.mjs';
|
|
13
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
|
|
14
|
-
import { getNestedValue } from './chunk-DQNKVOJ2.mjs';
|
|
15
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KGOXHLLZ.mjs';
|
|
16
|
-
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
17
|
-
import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
|
|
18
|
-
import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
|
|
19
|
-
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
20
|
-
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
|
+
import { getLanguage, withOpenIndex, oneAtATime } from './chunk-2HJWQPR6.mjs';
|
|
12
|
+
import { loadVectorCodeFor, loadedVectorCode, loadVectorCode, requireLoadedVectorCode } from './chunk-ZAYWOHVG.mjs';
|
|
21
13
|
import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
|
|
22
|
-
import {
|
|
14
|
+
import { collectAndReturnPages } from './chunk-QYRNLLLI.mjs';
|
|
15
|
+
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXNQEW43.mjs';
|
|
16
|
+
import { requireSearchBackendSettings } from './chunk-NVTFQFUL.mjs';
|
|
17
|
+
import { countWrite, turnsAreBudgeted, turnIsSpent, yieldTurn, indexChangeOutsideExternalWaits, changeIndexOutsideExternalWaits } from './chunk-LQG5G2LY.mjs';
|
|
23
18
|
import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
24
|
-
import {
|
|
19
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-U46IZNCW.mjs';
|
|
20
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, getNestedValue, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-SOUZMPME.mjs';
|
|
21
|
+
import { isSingleStringFieldType } from './chunk-KYVRZAYP.mjs';
|
|
22
|
+
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
23
|
+
import { ErrorCodes, NarsilError } from './chunk-MDF6PDIA.mjs';
|
|
24
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
25
25
|
import { encode, decode } from '@msgpack/msgpack';
|
|
26
26
|
import * as zlib from 'zlib';
|
|
27
27
|
|
|
28
|
+
// src/engine/memory/fractions.ts
|
|
29
|
+
function memoryFractionsOf(config) {
|
|
30
|
+
const block = config?.blockWritesAbove ?? DEFAULT_BLOCK_WRITES_ABOVE;
|
|
31
|
+
const resume = config?.resumeWritesBelow ?? DEFAULT_RESUME_WRITES_BELOW;
|
|
32
|
+
return {
|
|
33
|
+
processBlock: block,
|
|
34
|
+
processResume: resume,
|
|
35
|
+
heapBlock: Math.min(block, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE),
|
|
36
|
+
heapResume: Math.min(resume, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE)
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// src/engine/memory/monitor.ts
|
|
41
|
+
function fractionOf(usedBytes, limitBytes) {
|
|
42
|
+
return limitBytes > 0 ? usedBytes / limitBytes : 0;
|
|
43
|
+
}
|
|
44
|
+
function reportOf(figures, full) {
|
|
45
|
+
return {
|
|
46
|
+
processBytes: figures.processBytes,
|
|
47
|
+
processLimitBytes: figures.processLimitBytes,
|
|
48
|
+
heapBytes: figures.heapBytes,
|
|
49
|
+
heapLimitBytes: figures.heapLimitBytes,
|
|
50
|
+
full
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
function atOrAboveResume(figures, fractions) {
|
|
54
|
+
return fractionOf(figures.processBytes + figures.youngReserveBytes, figures.processLimitBytes) >= fractions.processResume || fractionOf(figures.heapBytes, figures.heapLimitBytes) >= fractions.heapResume;
|
|
55
|
+
}
|
|
56
|
+
function nextFullState(wasFull, figures, fractions) {
|
|
57
|
+
if (wasFull) return atOrAboveResume(figures, fractions);
|
|
58
|
+
return fractionOf(figures.processBytes + figures.youngReserveBytes, figures.processLimitBytes) > fractions.processBlock || fractionOf(figures.heapBytes, figures.heapLimitBytes) > fractions.heapBlock;
|
|
59
|
+
}
|
|
60
|
+
function createMemoryMonitor(options) {
|
|
61
|
+
let latest = null;
|
|
62
|
+
let timer = null;
|
|
63
|
+
let lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
64
|
+
let lastSampleAt = Number.NEGATIVE_INFINITY;
|
|
65
|
+
let collectionCount = 0;
|
|
66
|
+
let latestMeasured = null;
|
|
67
|
+
function figuresWithoutGarbageBeforeRefusal(figures) {
|
|
68
|
+
if (!options.collectGarbageBeforeRefusal()) {
|
|
69
|
+
return options.collectYoungGenerationBeforeRefusal() ? options.read() ?? figures : figures;
|
|
70
|
+
}
|
|
71
|
+
collectionCount++;
|
|
72
|
+
return options.read() ?? figures;
|
|
73
|
+
}
|
|
74
|
+
function figuresWithoutGarbage(figures) {
|
|
75
|
+
const now = Date.now();
|
|
76
|
+
if (now - lastCollectionAt < options.collectionIntervalMs) {
|
|
77
|
+
return figures;
|
|
78
|
+
}
|
|
79
|
+
lastCollectionAt = now;
|
|
80
|
+
if (!options.collectGarbage()) {
|
|
81
|
+
return figures;
|
|
82
|
+
}
|
|
83
|
+
collectionCount++;
|
|
84
|
+
return options.read() ?? figures;
|
|
85
|
+
}
|
|
86
|
+
function record(figures) {
|
|
87
|
+
const wasFull = latest?.full ?? false;
|
|
88
|
+
let full = nextFullState(wasFull, figures, options.fractions);
|
|
89
|
+
let settled2 = figures;
|
|
90
|
+
if (full) {
|
|
91
|
+
settled2 = figuresWithoutGarbage(figures);
|
|
92
|
+
full = nextFullState(wasFull, settled2, options.fractions);
|
|
93
|
+
}
|
|
94
|
+
latestMeasured = settled2;
|
|
95
|
+
latest = reportOf(settled2, full);
|
|
96
|
+
if (full !== wasFull) {
|
|
97
|
+
options.onFullChange(full);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function sample() {
|
|
101
|
+
lastSampleAt = Date.now();
|
|
102
|
+
const figures = options.read();
|
|
103
|
+
if (figures !== null) {
|
|
104
|
+
record(figures);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function latestFigures() {
|
|
108
|
+
return latestMeasured;
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
report: () => latest,
|
|
112
|
+
isFull: () => latest?.full ?? false,
|
|
113
|
+
sample,
|
|
114
|
+
measureWithoutGarbage() {
|
|
115
|
+
lastSampleAt = Date.now();
|
|
116
|
+
const figures = options.read();
|
|
117
|
+
if (figures === null) return latestFigures();
|
|
118
|
+
record(figuresWithoutGarbageBeforeRefusal(figures));
|
|
119
|
+
return latestFigures();
|
|
120
|
+
},
|
|
121
|
+
refusedAfterCollection(figures) {
|
|
122
|
+
if (latest?.full === true) return;
|
|
123
|
+
if (!atOrAboveResume(figures, options.fractions)) return;
|
|
124
|
+
latestMeasured = { ...figures };
|
|
125
|
+
latest = reportOf(figures, true);
|
|
126
|
+
options.onFullChange(true);
|
|
127
|
+
},
|
|
128
|
+
remeasureAfterRelease() {
|
|
129
|
+
lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
130
|
+
lastSampleAt = Number.NEGATIVE_INFINITY;
|
|
131
|
+
},
|
|
132
|
+
collections: () => collectionCount,
|
|
133
|
+
sampleIfOlderThan(maxAgeMs) {
|
|
134
|
+
if (timer !== null && Date.now() - lastSampleAt >= maxAgeMs) {
|
|
135
|
+
sample();
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
measureIfOlderThan(maxAgeMs) {
|
|
139
|
+
if (Date.now() - lastSampleAt >= maxAgeMs) {
|
|
140
|
+
sample();
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
start() {
|
|
144
|
+
if (timer !== null) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
sample();
|
|
148
|
+
timer = setInterval(sample, options.intervalMs);
|
|
149
|
+
timer.unref?.();
|
|
150
|
+
},
|
|
151
|
+
stop() {
|
|
152
|
+
if (timer !== null) {
|
|
153
|
+
clearInterval(timer);
|
|
154
|
+
timer = null;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// src/engine/memory/measured-block.ts
|
|
161
|
+
var MEMORY_FULL_REASON = "memory_full";
|
|
162
|
+
function finiteOrNull(value) {
|
|
163
|
+
return value !== void 0 && Number.isFinite(value) ? value : null;
|
|
164
|
+
}
|
|
165
|
+
function createMeasuredWriteBlock(settings) {
|
|
166
|
+
const fractions = memoryFractionsOf(settings);
|
|
167
|
+
const monitor = createMemoryMonitor({
|
|
168
|
+
fractions,
|
|
169
|
+
intervalMs: ENGINE_MEMORY_FIGURES_MAX_AGE_MS,
|
|
170
|
+
collectionIntervalMs: FULL_MEMORY_COLLECTION_INTERVAL_MS,
|
|
171
|
+
read: readEngineMemoryFigures,
|
|
172
|
+
collectGarbage: collectAndReturnPages,
|
|
173
|
+
collectGarbageBeforeRefusal,
|
|
174
|
+
collectYoungGenerationBeforeRefusal,
|
|
175
|
+
onFullChange: () => {
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
return {
|
|
179
|
+
fractions,
|
|
180
|
+
monitor,
|
|
181
|
+
refuseWhileFull(indexName) {
|
|
182
|
+
monitor.measureIfOlderThan(ENGINE_MEMORY_FIGURES_MAX_AGE_MS);
|
|
183
|
+
if (!monitor.isFull()) return;
|
|
184
|
+
monitor.measureWithoutGarbage();
|
|
185
|
+
if (!monitor.isFull()) return;
|
|
186
|
+
const report = monitor.report();
|
|
187
|
+
throw new NarsilError(
|
|
188
|
+
ErrorCodes.MEMORY_WRITE_BLOCKED,
|
|
189
|
+
`The engine refuses inserts and updates to index '${indexName}' because its memory passed the block level of its limit. Remove documents to free memory, and writes resume once memory falls below the resume level.`,
|
|
190
|
+
{
|
|
191
|
+
reason: MEMORY_FULL_REASON,
|
|
192
|
+
indexName,
|
|
193
|
+
processBytes: finiteOrNull(report?.processBytes),
|
|
194
|
+
processLimitBytes: finiteOrNull(report?.processLimitBytes),
|
|
195
|
+
heapBytes: finiteOrNull(report?.heapBytes),
|
|
196
|
+
heapLimitBytes: finiteOrNull(report?.heapLimitBytes)
|
|
197
|
+
}
|
|
198
|
+
);
|
|
199
|
+
},
|
|
200
|
+
memoryReleased() {
|
|
201
|
+
if (monitor.isFull()) monitor.remeasureAfterRelease();
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// src/engine/memory/validate.ts
|
|
207
|
+
function requireFraction(setting, value) {
|
|
208
|
+
if (value === void 0) return;
|
|
209
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1) return;
|
|
210
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${setting} must be a number above 0 and at most 1`, {
|
|
211
|
+
[setting.slice(setting.lastIndexOf(".") + 1)]: value
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
function validateMemoryFractions(section, memory, extraFractions = []) {
|
|
215
|
+
if (memory === void 0) return;
|
|
216
|
+
if (typeof memory !== "object" || memory === null || Array.isArray(memory)) {
|
|
217
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${section} must be an object`, { memory });
|
|
218
|
+
}
|
|
219
|
+
const settings = memory;
|
|
220
|
+
for (const name of ["blockWritesAbove", "resumeWritesBelow", ...extraFractions]) {
|
|
221
|
+
requireFraction(`${section}.${name}`, settings[name]);
|
|
222
|
+
}
|
|
223
|
+
const block = typeof settings.blockWritesAbove === "number" ? settings.blockWritesAbove : DEFAULT_BLOCK_WRITES_ABOVE;
|
|
224
|
+
const resume = typeof settings.resumeWritesBelow === "number" ? settings.resumeWritesBelow : DEFAULT_RESUME_WRITES_BELOW;
|
|
225
|
+
if (resume >= block) {
|
|
226
|
+
throw new NarsilError(
|
|
227
|
+
ErrorCodes.CONFIG_INVALID,
|
|
228
|
+
`${section}.resumeWritesBelow (${resume}) must lie below blockWritesAbove (${block})`,
|
|
229
|
+
{ blockWritesAbove: block, resumeWritesBelow: resume }
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// src/engine/memory/write-block.ts
|
|
235
|
+
var MEMORY_CONFIG_KEYS = {
|
|
236
|
+
blockWritesAbove: true,
|
|
237
|
+
resumeWritesBelow: true
|
|
238
|
+
};
|
|
239
|
+
function validateMemorySettings(settings) {
|
|
240
|
+
if (settings !== null && typeof settings === "object") {
|
|
241
|
+
for (const key of Object.keys(settings)) {
|
|
242
|
+
if (Object.hasOwn(MEMORY_CONFIG_KEYS, key)) continue;
|
|
243
|
+
throw new NarsilError(
|
|
244
|
+
ErrorCodes.CONFIG_INVALID,
|
|
245
|
+
`The configuration sets "memory.${key}", which is no setting that createNarsil accepts. The settings at that level are ${Object.keys(MEMORY_CONFIG_KEYS).join(", ")}`,
|
|
246
|
+
{ setting: `memory.${key}`, settings: Object.keys(MEMORY_CONFIG_KEYS) }
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
validateMemoryFractions("memory", settings);
|
|
251
|
+
}
|
|
252
|
+
function createMemoryWriteBlock(deps) {
|
|
253
|
+
if (deps.capacityScope !== "index") return void 0;
|
|
254
|
+
const measured = createMeasuredWriteBlock(deps.settings);
|
|
255
|
+
return { refuseWhileFull: measured.refuseWhileFull, memoryReleased: measured.memoryReleased };
|
|
256
|
+
}
|
|
257
|
+
|
|
28
258
|
// src/engine/config-keys.ts
|
|
29
259
|
var NARSIL_CONFIG_KEYS = {
|
|
30
260
|
persistence: true,
|
|
@@ -41,7 +271,8 @@ var NARSIL_CONFIG_KEYS = {
|
|
|
41
271
|
analysis: true,
|
|
42
272
|
lifecycle: true,
|
|
43
273
|
patternWorkCap: true,
|
|
44
|
-
events: true
|
|
274
|
+
events: true,
|
|
275
|
+
memory: true
|
|
45
276
|
};
|
|
46
277
|
var EVENT_NAMES = {
|
|
47
278
|
write: true,
|
|
@@ -119,6 +350,7 @@ function requireKnownConfig(config) {
|
|
|
119
350
|
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
120
351
|
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
121
352
|
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
353
|
+
validateMemorySettings(config.memory);
|
|
122
354
|
requireNamedPlugins(config.plugins);
|
|
123
355
|
requireValidPatternWorkCap(config.patternWorkCap);
|
|
124
356
|
requireEventListeners(config.events);
|
|
@@ -376,7 +608,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
376
608
|
// src/partitioning/rewrite-spill-of.ts
|
|
377
609
|
async function rewriteSpillOf(storage, manager, target) {
|
|
378
610
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
379
|
-
const { openRewriteSpill } = await import('./module-
|
|
611
|
+
const { openRewriteSpill } = await import('./module-V5K4ZP23.mjs');
|
|
380
612
|
return openRewriteSpill(storage, manager, target);
|
|
381
613
|
}
|
|
382
614
|
|
|
@@ -920,7 +1152,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
920
1152
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
921
1153
|
if (existing !== void 0) {
|
|
922
1154
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
923
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
1155
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-ZCJNYBJM.mjs');
|
|
924
1156
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
925
1157
|
}
|
|
926
1158
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -2275,6 +2507,10 @@ function markCopiesLoaded(state, indexName, copyLoad) {
|
|
|
2275
2507
|
requestId: `copy-loaded-${copyLoad}`
|
|
2276
2508
|
});
|
|
2277
2509
|
}
|
|
2510
|
+
function copiesWillReceive(state, indexName) {
|
|
2511
|
+
if (state.copyLoadBuffers.has(indexName) || state.replicationQueues.has(indexName)) return true;
|
|
2512
|
+
return state.workerPool !== null && state.scaledOutIndexes.has(indexName);
|
|
2513
|
+
}
|
|
2278
2514
|
async function replicateToWorkers(state, action) {
|
|
2279
2515
|
if (!("indexName" in action)) {
|
|
2280
2516
|
if (state.workerPool === null) return;
|
|
@@ -2282,6 +2518,7 @@ async function replicateToWorkers(state, action) {
|
|
|
2282
2518
|
await dispatchToWorkers(state, action);
|
|
2283
2519
|
return;
|
|
2284
2520
|
}
|
|
2521
|
+
if (!copiesWillReceive(state, action.indexName)) return;
|
|
2285
2522
|
const detached = detachCallerDocuments(state, action);
|
|
2286
2523
|
const loading = state.copyLoadBuffers.get(action.indexName);
|
|
2287
2524
|
if (loading !== void 0) {
|
|
@@ -2771,297 +3008,6 @@ async function createWorkerFactory(entryPoint) {
|
|
|
2771
3008
|
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
|
|
2772
3009
|
}
|
|
2773
3010
|
|
|
2774
|
-
// src/workers/pool-memory.ts
|
|
2775
|
-
function toFinite(value) {
|
|
2776
|
-
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
2777
|
-
}
|
|
2778
|
-
function toHeapLimit(value) {
|
|
2779
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
2780
|
-
}
|
|
2781
|
-
async function memoryStatsOf(slots) {
|
|
2782
|
-
const indices = [];
|
|
2783
|
-
const pending = [];
|
|
2784
|
-
for (const [slotIndex, slot] of slots) {
|
|
2785
|
-
indices.push(slotIndex);
|
|
2786
|
-
pending.push(
|
|
2787
|
-
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
2788
|
-
);
|
|
2789
|
-
}
|
|
2790
|
-
const reports = await Promise.all(pending);
|
|
2791
|
-
const stats = [];
|
|
2792
|
-
for (let n = 0; n < indices.length; n++) {
|
|
2793
|
-
const report = reports[n];
|
|
2794
|
-
stats.push({
|
|
2795
|
-
workerId: indices[n],
|
|
2796
|
-
heapUsed: toFinite(report?.heapUsed),
|
|
2797
|
-
heapTotal: toFinite(report?.heapTotal),
|
|
2798
|
-
heapLimit: toHeapLimit(report?.heapLimit),
|
|
2799
|
-
external: toFinite(report?.external)
|
|
2800
|
-
});
|
|
2801
|
-
}
|
|
2802
|
-
return stats;
|
|
2803
|
-
}
|
|
2804
|
-
|
|
2805
|
-
// src/workers/pool.ts
|
|
2806
|
-
function createWorkerPool(config) {
|
|
2807
|
-
const workerCount = resolveWorkerCount(config.count);
|
|
2808
|
-
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
2809
|
-
const workers = /* @__PURE__ */ new Map();
|
|
2810
|
-
const deadSlots = /* @__PURE__ */ new Set();
|
|
2811
|
-
const goneSlots = /* @__PURE__ */ new Set();
|
|
2812
|
-
const threadsStillRunning = /* @__PURE__ */ new Set();
|
|
2813
|
-
const waitingForEveryThread = [];
|
|
2814
|
-
const indexAssignment = /* @__PURE__ */ new Map();
|
|
2815
|
-
let isShutdown = false;
|
|
2816
|
-
function handleWorkerDeath(slotIndex, error) {
|
|
2817
|
-
if (isShutdown) {
|
|
2818
|
-
return;
|
|
2819
|
-
}
|
|
2820
|
-
const slot = workers.get(slotIndex);
|
|
2821
|
-
if (slot !== void 0 && !slot.serving) {
|
|
2822
|
-
workers.delete(slotIndex);
|
|
2823
|
-
goneSlots.delete(slotIndex);
|
|
2824
|
-
return;
|
|
2825
|
-
}
|
|
2826
|
-
if (deadSlots.has(slotIndex)) {
|
|
2827
|
-
return;
|
|
2828
|
-
}
|
|
2829
|
-
deadSlots.add(slotIndex);
|
|
2830
|
-
const indexNames = slot ? [...slot.indexes].sort() : [];
|
|
2831
|
-
workers.delete(slotIndex);
|
|
2832
|
-
config.onWorkerCrash?.(slotIndex, indexNames, error);
|
|
2833
|
-
}
|
|
2834
|
-
function handleWorkerGone(slotIndex) {
|
|
2835
|
-
if (!deadSlots.has(slotIndex) || workers.has(slotIndex)) {
|
|
2836
|
-
return;
|
|
2837
|
-
}
|
|
2838
|
-
goneSlots.add(slotIndex);
|
|
2839
|
-
config.onWorkerGone?.(slotIndex);
|
|
2840
|
-
}
|
|
2841
|
-
function spawnSlot(slotIndex, serving) {
|
|
2842
|
-
const slot = {
|
|
2843
|
-
executor: config.workerFactory(
|
|
2844
|
-
slotIndex,
|
|
2845
|
-
(error) => handleWorkerDeath(slotIndex, error),
|
|
2846
|
-
() => {
|
|
2847
|
-
noteThreadGone(slot.executor);
|
|
2848
|
-
handleWorkerGone(slotIndex);
|
|
2849
|
-
}
|
|
2850
|
-
),
|
|
2851
|
-
indexes: /* @__PURE__ */ new Set(),
|
|
2852
|
-
inFlight: 0,
|
|
2853
|
-
serving
|
|
2854
|
-
};
|
|
2855
|
-
threadsStillRunning.add(slot.executor);
|
|
2856
|
-
workers.set(slotIndex, slot);
|
|
2857
|
-
return slot;
|
|
2858
|
-
}
|
|
2859
|
-
function noteThreadGone(executor) {
|
|
2860
|
-
threadsStillRunning.delete(executor);
|
|
2861
|
-
if (threadsStillRunning.size > 0) return;
|
|
2862
|
-
for (const resume of waitingForEveryThread.splice(0)) resume();
|
|
2863
|
-
}
|
|
2864
|
-
function whenEveryThreadIsGone() {
|
|
2865
|
-
if (threadsStillRunning.size === 0) return Promise.resolve();
|
|
2866
|
-
return new Promise((resolve) => {
|
|
2867
|
-
waitingForEveryThread.push(resolve);
|
|
2868
|
-
});
|
|
2869
|
-
}
|
|
2870
|
-
function shutDownUntilTheThreadIsGone(executor) {
|
|
2871
|
-
return executor.shutdown().then(() => noteThreadGone(executor));
|
|
2872
|
-
}
|
|
2873
|
-
function ensureWorker(slotIndex) {
|
|
2874
|
-
const slot = workers.get(slotIndex);
|
|
2875
|
-
if (slot !== void 0) return slot;
|
|
2876
|
-
if (deadSlots.has(slotIndex)) return void 0;
|
|
2877
|
-
return spawnSlot(slotIndex, true);
|
|
2878
|
-
}
|
|
2879
|
-
function deadWorkerIds() {
|
|
2880
|
-
return [...deadSlots].filter((slotIndex) => goneSlots.has(slotIndex) && !workers.has(slotIndex)).sort((a, b) => a - b);
|
|
2881
|
-
}
|
|
2882
|
-
function spawnReplacement(slotIndex) {
|
|
2883
|
-
if (isShutdown || !deadSlots.has(slotIndex) || !goneSlots.has(slotIndex) || workers.has(slotIndex)) return null;
|
|
2884
|
-
const slot = spawnSlot(slotIndex, false);
|
|
2885
|
-
return {
|
|
2886
|
-
workerId: slotIndex,
|
|
2887
|
-
executor: slot.executor,
|
|
2888
|
-
hold(indexName) {
|
|
2889
|
-
slot.indexes.add(indexName);
|
|
2890
|
-
},
|
|
2891
|
-
admit() {
|
|
2892
|
-
if (workers.get(slotIndex) !== slot) return;
|
|
2893
|
-
deadSlots.delete(slotIndex);
|
|
2894
|
-
goneSlots.delete(slotIndex);
|
|
2895
|
-
slot.serving = true;
|
|
2896
|
-
},
|
|
2897
|
-
abandon() {
|
|
2898
|
-
if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
|
|
2899
|
-
void shutDownUntilTheThreadIsGone(slot.executor).catch(() => void 0);
|
|
2900
|
-
}
|
|
2901
|
-
};
|
|
2902
|
-
}
|
|
2903
|
-
function isLongQueryWorker(workerId) {
|
|
2904
|
-
return workerId >= queryWorkerCount;
|
|
2905
|
-
}
|
|
2906
|
-
function assignSlot(indexName) {
|
|
2907
|
-
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
2908
|
-
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
2909
|
-
const candidate2 = (preferred + probe) % queryWorkerCount;
|
|
2910
|
-
if (!deadSlots.has(candidate2)) {
|
|
2911
|
-
return candidate2;
|
|
2912
|
-
}
|
|
2913
|
-
}
|
|
2914
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
|
|
2915
|
-
}
|
|
2916
|
-
function addIndex(indexName) {
|
|
2917
|
-
if (isShutdown) {
|
|
2918
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
2919
|
-
}
|
|
2920
|
-
if (indexAssignment.has(indexName)) {
|
|
2921
|
-
return;
|
|
2922
|
-
}
|
|
2923
|
-
const slotIndex = assignSlot(indexName);
|
|
2924
|
-
const slot = ensureWorker(slotIndex);
|
|
2925
|
-
if (!slot) {
|
|
2926
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
|
|
2927
|
-
}
|
|
2928
|
-
slot.indexes.add(indexName);
|
|
2929
|
-
indexAssignment.set(indexName, slotIndex);
|
|
2930
|
-
}
|
|
2931
|
-
function removeIndex(indexName) {
|
|
2932
|
-
for (const slot of workers.values()) slot.indexes.delete(indexName);
|
|
2933
|
-
indexAssignment.delete(indexName);
|
|
2934
|
-
}
|
|
2935
|
-
function getExecutor(indexName) {
|
|
2936
|
-
if (isShutdown) {
|
|
2937
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
2938
|
-
}
|
|
2939
|
-
const slotIndex = indexAssignment.get(indexName);
|
|
2940
|
-
if (slotIndex === void 0) {
|
|
2941
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
2942
|
-
}
|
|
2943
|
-
const slot = workers.get(slotIndex);
|
|
2944
|
-
if (slot?.serving) {
|
|
2945
|
-
return slot.executor;
|
|
2946
|
-
}
|
|
2947
|
-
for (const survivor of workers.values()) {
|
|
2948
|
-
if (survivor.serving && survivor.indexes.has(indexName)) {
|
|
2949
|
-
return survivor.executor;
|
|
2950
|
-
}
|
|
2951
|
-
}
|
|
2952
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
2953
|
-
}
|
|
2954
|
-
async function shutdown() {
|
|
2955
|
-
if (isShutdown) {
|
|
2956
|
-
return;
|
|
2957
|
-
}
|
|
2958
|
-
isShutdown = true;
|
|
2959
|
-
const shutdownPromises = [...threadsStillRunning].map((executor) => {
|
|
2960
|
-
let timeoutId;
|
|
2961
|
-
const timeoutPromise = new Promise((resolve) => {
|
|
2962
|
-
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
2963
|
-
});
|
|
2964
|
-
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
2965
|
-
() => clearTimeout(timeoutId)
|
|
2966
|
-
);
|
|
2967
|
-
});
|
|
2968
|
-
await Promise.allSettled(shutdownPromises);
|
|
2969
|
-
indexAssignment.clear();
|
|
2970
|
-
}
|
|
2971
|
-
function getAllExecutors() {
|
|
2972
|
-
return [...workers.values()].map((slot) => slot.executor);
|
|
2973
|
-
}
|
|
2974
|
-
function executorEntries() {
|
|
2975
|
-
return [...workers].map(([workerId, slot]) => ({ workerId, executor: slot.executor }));
|
|
2976
|
-
}
|
|
2977
|
-
function executorsHolding(indexName) {
|
|
2978
|
-
const holders = [];
|
|
2979
|
-
for (const slot of workers.values()) {
|
|
2980
|
-
if (slot.indexes.has(indexName)) holders.push(slot.executor);
|
|
2981
|
-
}
|
|
2982
|
-
return holders;
|
|
2983
|
-
}
|
|
2984
|
-
function lease(workerId, slot) {
|
|
2985
|
-
slot.inFlight += 1;
|
|
2986
|
-
let released = false;
|
|
2987
|
-
return {
|
|
2988
|
-
workerId,
|
|
2989
|
-
executor: slot.executor,
|
|
2990
|
-
release() {
|
|
2991
|
-
if (released) return;
|
|
2992
|
-
released = true;
|
|
2993
|
-
slot.inFlight -= 1;
|
|
2994
|
-
}
|
|
2995
|
-
};
|
|
2996
|
-
}
|
|
2997
|
-
function leaseLeastBusy(indexName) {
|
|
2998
|
-
let chosenId = -1;
|
|
2999
|
-
let chosen = null;
|
|
3000
|
-
for (const [workerId, slot] of workers) {
|
|
3001
|
-
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
3002
|
-
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
3003
|
-
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
3004
|
-
chosenId = workerId;
|
|
3005
|
-
chosen = slot;
|
|
3006
|
-
}
|
|
3007
|
-
if (chosen.inFlight === 0) break;
|
|
3008
|
-
}
|
|
3009
|
-
return chosen === null ? null : lease(chosenId, chosen);
|
|
3010
|
-
}
|
|
3011
|
-
function queriesInFlight() {
|
|
3012
|
-
let total = 0;
|
|
3013
|
-
for (const [workerId, slot] of workers) {
|
|
3014
|
-
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
3015
|
-
}
|
|
3016
|
-
return total;
|
|
3017
|
-
}
|
|
3018
|
-
function leaseIdle(limit) {
|
|
3019
|
-
const leases = [];
|
|
3020
|
-
for (const [workerId, slot] of workers) {
|
|
3021
|
-
if (leases.length >= limit) break;
|
|
3022
|
-
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
3023
|
-
}
|
|
3024
|
-
return leases;
|
|
3025
|
-
}
|
|
3026
|
-
function addIndexToAll(indexName) {
|
|
3027
|
-
if (isShutdown) {
|
|
3028
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
3029
|
-
}
|
|
3030
|
-
for (let i = 0; i < workerCount; i++) {
|
|
3031
|
-
ensureWorker(i)?.indexes.add(indexName);
|
|
3032
|
-
}
|
|
3033
|
-
indexAssignment.set(indexName, assignSlot(indexName));
|
|
3034
|
-
}
|
|
3035
|
-
function spawnAll() {
|
|
3036
|
-
if (isShutdown) {
|
|
3037
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
3038
|
-
}
|
|
3039
|
-
for (let i = 0; i < workerCount; i++) ensureWorker(i);
|
|
3040
|
-
}
|
|
3041
|
-
return {
|
|
3042
|
-
getExecutor,
|
|
3043
|
-
getAllExecutors,
|
|
3044
|
-
executorEntries,
|
|
3045
|
-
executorsHolding,
|
|
3046
|
-
leaseLeastBusy,
|
|
3047
|
-
leaseIdle,
|
|
3048
|
-
isLongQueryWorker,
|
|
3049
|
-
queriesInFlight,
|
|
3050
|
-
spawnAll,
|
|
3051
|
-
deadWorkerIds,
|
|
3052
|
-
spawnReplacement,
|
|
3053
|
-
get workerCount() {
|
|
3054
|
-
return workerCount;
|
|
3055
|
-
},
|
|
3056
|
-
addIndex,
|
|
3057
|
-
addIndexToAll,
|
|
3058
|
-
removeIndex,
|
|
3059
|
-
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
3060
|
-
shutdown,
|
|
3061
|
-
whenEveryThreadIsGone
|
|
3062
|
-
};
|
|
3063
|
-
}
|
|
3064
|
-
|
|
3065
3011
|
// src/engine/orchestration/vector-copies.ts
|
|
3066
3012
|
function holdsIndex(state, indexName) {
|
|
3067
3013
|
return state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName);
|
|
@@ -3351,7 +3297,7 @@ async function startPool(state) {
|
|
|
3351
3297
|
);
|
|
3352
3298
|
}
|
|
3353
3299
|
eligibleIndexNames(state);
|
|
3354
|
-
const factory = await createWorkerFactory();
|
|
3300
|
+
const [factory, { createWorkerPool }] = await Promise.all([createWorkerFactory(), import('./pool-UPV5PVPY.mjs')]);
|
|
3355
3301
|
let started = null;
|
|
3356
3302
|
const pool = createWorkerPool({
|
|
3357
3303
|
count: state.keywordWorkerCount + state.longQueryWorkerCount,
|
|
@@ -4042,7 +3988,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
4042
3988
|
// src/engine/core.ts
|
|
4043
3989
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
4044
3990
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
4045
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
3991
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-L2DZYY6N.mjs')).resolveRerankSettings(config);
|
|
4046
3992
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
4047
3993
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
4048
3994
|
const vectorCopyPolicy = {
|
|
@@ -4053,7 +3999,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4053
3999
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4054
4000
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4055
4001
|
}
|
|
4056
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4002
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-GNWQSZI3.mjs');
|
|
4057
4003
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4058
4004
|
const executor = createDirectExecutor({
|
|
4059
4005
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4238,7 +4184,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4238
4184
|
requireManager,
|
|
4239
4185
|
resumeChange: async (indexName) => {
|
|
4240
4186
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
4241
|
-
await (await import('./analysis-change-entry-
|
|
4187
|
+
await (await import('./analysis-change-entry-YN3ELL6C.mjs')).resumeChangeOf(engineCore, indexName);
|
|
4242
4188
|
},
|
|
4243
4189
|
onClose: hooks?.onIndexClose,
|
|
4244
4190
|
onOpen: (indexName) => {
|
|
@@ -4265,7 +4211,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4265
4211
|
bufferedDocState: (indexName, docId) => waqMap.get(indexName)?.bufferedDocState(docId),
|
|
4266
4212
|
capacityScope,
|
|
4267
4213
|
checkWatermark: watermarkNotifier.check,
|
|
4268
|
-
checkHeapPressure: heapPressureNotifier.check
|
|
4214
|
+
checkHeapPressure: heapPressureNotifier.check,
|
|
4215
|
+
memoryWriteBlock: createMemoryWriteBlock({
|
|
4216
|
+
settings: config?.memory,
|
|
4217
|
+
capacityScope})
|
|
4269
4218
|
};
|
|
4270
4219
|
const rebalanceCtx = {
|
|
4271
4220
|
router: rebalanceRouter,
|
|
@@ -4320,7 +4269,7 @@ async function startEngineCore(core, config) {
|
|
|
4320
4269
|
checkHeapAfterRecovery(core);
|
|
4321
4270
|
}
|
|
4322
4271
|
if (core.invalidation) await core.invalidation.start();
|
|
4323
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
4272
|
+
if (core.durability) await (await import('./analysis-change-entry-YN3ELL6C.mjs')).resumeEveryChange(core);
|
|
4324
4273
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
4325
4274
|
} catch (err) {
|
|
4326
4275
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -4358,7 +4307,7 @@ async function shutdownEngine(core) {
|
|
|
4358
4307
|
}
|
|
4359
4308
|
}
|
|
4360
4309
|
if (core.rerankAdapters.size > 0) {
|
|
4361
|
-
await (await import('./runtime-
|
|
4310
|
+
await (await import('./runtime-L2DZYY6N.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
4362
4311
|
}
|
|
4363
4312
|
await orchestrator.shutdown();
|
|
4364
4313
|
await executor.shutdown();
|
|
@@ -4597,34 +4546,17 @@ function loadFailed(message, details) {
|
|
|
4597
4546
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
|
|
4598
4547
|
}
|
|
4599
4548
|
async function gunzipWithinLimit(compressed, what) {
|
|
4600
|
-
const
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
let step;
|
|
4606
|
-
try {
|
|
4607
|
-
step = await reader.read();
|
|
4608
|
-
} catch (error) {
|
|
4609
|
-
loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
|
|
4610
|
-
cause: error instanceof Error ? error.message : String(error)
|
|
4611
|
-
});
|
|
4612
|
-
}
|
|
4613
|
-
if (step.done) break;
|
|
4614
|
-
total += step.value.byteLength;
|
|
4615
|
-
if (total > MAX_FILE_BYTES) {
|
|
4616
|
-
await reader.cancel();
|
|
4549
|
+
const { gunzip } = await import('./gzip-F4HMIEEQ.mjs');
|
|
4550
|
+
try {
|
|
4551
|
+
return await gunzip(compressed, MAX_FILE_BYTES);
|
|
4552
|
+
} catch (error) {
|
|
4553
|
+
if (error?.code === "EFBIG") {
|
|
4617
4554
|
loadFailed(`The ${what} inflates past the file size limit of ${MAX_FILE_BYTES} bytes`);
|
|
4618
4555
|
}
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
let offset = 0;
|
|
4623
|
-
for (const chunk of chunks) {
|
|
4624
|
-
inflated.set(chunk, offset);
|
|
4625
|
-
offset += chunk.byteLength;
|
|
4556
|
+
loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
|
|
4557
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
4558
|
+
});
|
|
4626
4559
|
}
|
|
4627
|
-
return inflated;
|
|
4628
4560
|
}
|
|
4629
4561
|
async function openChecksummedEnvelope(data, expected) {
|
|
4630
4562
|
const { envelopeFormatVersion, what } = expected;
|
|
@@ -4924,7 +4856,7 @@ async function dropEngineIndex(core, name) {
|
|
|
4924
4856
|
const entry = core.requireIndex(name);
|
|
4925
4857
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
4926
4858
|
const abandonChange = async () => {
|
|
4927
|
-
await (await import('./analysis-change-entry-
|
|
4859
|
+
await (await import('./analysis-change-entry-YN3ELL6C.mjs')).abandonChangeOf(core, name);
|
|
4928
4860
|
};
|
|
4929
4861
|
if (changeRecorded()) await abandonChange();
|
|
4930
4862
|
await core.indexState.drop(name, async () => {
|
|
@@ -4975,6 +4907,224 @@ function registerEngineEmbeddingAdapter(core, name, adapter) {
|
|
|
4975
4907
|
}
|
|
4976
4908
|
}
|
|
4977
4909
|
|
|
4910
|
+
// src/engine/embed.ts
|
|
4911
|
+
function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
|
|
4912
|
+
for (const targetField of Object.keys(embeddingConfig.fields)) {
|
|
4913
|
+
const existing = resolveFieldValue(document, targetField);
|
|
4914
|
+
if (existing === void 0 || existing === null) {
|
|
4915
|
+
throw new NarsilError(
|
|
4916
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4917
|
+
adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
|
|
4918
|
+
adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
|
|
4919
|
+
);
|
|
4920
|
+
}
|
|
4921
|
+
}
|
|
4922
|
+
}
|
|
4923
|
+
function validateVector(vector, targetField, expectedDimensions) {
|
|
4924
|
+
if (!(vector instanceof Float32Array)) {
|
|
4925
|
+
throw new NarsilError(
|
|
4926
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4927
|
+
`Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
|
|
4928
|
+
{ field: targetField, returnedType: typeof vector }
|
|
4929
|
+
);
|
|
4930
|
+
}
|
|
4931
|
+
if (vector.length !== expectedDimensions) {
|
|
4932
|
+
throw new NarsilError(
|
|
4933
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
4934
|
+
`Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
|
|
4935
|
+
{ field: targetField, expected: expectedDimensions, actual: vector.length }
|
|
4936
|
+
);
|
|
4937
|
+
}
|
|
4938
|
+
for (let i = 0; i < vector.length; i++) {
|
|
4939
|
+
if (Number.isNaN(vector[i])) {
|
|
4940
|
+
throw new NarsilError(
|
|
4941
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4942
|
+
`Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
|
|
4943
|
+
{ field: targetField, index: i }
|
|
4944
|
+
);
|
|
4945
|
+
}
|
|
4946
|
+
}
|
|
4947
|
+
return vector;
|
|
4948
|
+
}
|
|
4949
|
+
function resolveFieldValue(document, path) {
|
|
4950
|
+
if (!path.includes(".")) return document[path];
|
|
4951
|
+
const segments = path.split(".");
|
|
4952
|
+
let current = document;
|
|
4953
|
+
for (const segment of segments) {
|
|
4954
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4955
|
+
current = current[segment];
|
|
4956
|
+
}
|
|
4957
|
+
return current;
|
|
4958
|
+
}
|
|
4959
|
+
function setFieldValue(document, path, value) {
|
|
4960
|
+
if (!path.includes(".")) {
|
|
4961
|
+
document[path] = value;
|
|
4962
|
+
return;
|
|
4963
|
+
}
|
|
4964
|
+
const segments = path.split(".");
|
|
4965
|
+
let current = document;
|
|
4966
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
4967
|
+
let next = current[segments[i]];
|
|
4968
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
4969
|
+
next = {};
|
|
4970
|
+
current[segments[i]] = next;
|
|
4971
|
+
}
|
|
4972
|
+
current = next;
|
|
4973
|
+
}
|
|
4974
|
+
current[segments[segments.length - 1]] = value;
|
|
4975
|
+
}
|
|
4976
|
+
function deleteFieldValue(document, path) {
|
|
4977
|
+
if (!path.includes(".")) {
|
|
4978
|
+
delete document[path];
|
|
4979
|
+
return;
|
|
4980
|
+
}
|
|
4981
|
+
const segments = path.split(".");
|
|
4982
|
+
let current = document;
|
|
4983
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
4984
|
+
const next = current[segments[i]];
|
|
4985
|
+
if (next === null || next === void 0 || typeof next !== "object") return;
|
|
4986
|
+
current = next;
|
|
4987
|
+
}
|
|
4988
|
+
delete current[segments[segments.length - 1]];
|
|
4989
|
+
}
|
|
4990
|
+
function collectSourceText(document, sourceFields, targetField, documentIndex) {
|
|
4991
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
4992
|
+
const parts = [];
|
|
4993
|
+
for (const source of sources) {
|
|
4994
|
+
const value = resolveFieldValue(document, source);
|
|
4995
|
+
if (value === void 0 || value === null || value === "") continue;
|
|
4996
|
+
if (typeof value !== "string") {
|
|
4997
|
+
throw new NarsilError(
|
|
4998
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4999
|
+
`Embedding source field "${source}" must be a string, got ${typeof value}`,
|
|
5000
|
+
{ field: source, targetField }
|
|
5001
|
+
);
|
|
5002
|
+
}
|
|
5003
|
+
parts.push(value);
|
|
5004
|
+
}
|
|
5005
|
+
if (parts.length === 0) {
|
|
5006
|
+
const details = { field: targetField };
|
|
5007
|
+
if (documentIndex !== void 0) {
|
|
5008
|
+
details.documentIndex = documentIndex;
|
|
5009
|
+
}
|
|
5010
|
+
throw new NarsilError(
|
|
5011
|
+
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
5012
|
+
`All source fields for embedding target "${targetField}" are missing or empty`,
|
|
5013
|
+
details
|
|
5014
|
+
);
|
|
5015
|
+
}
|
|
5016
|
+
return parts.join("\n");
|
|
5017
|
+
}
|
|
5018
|
+
function wrapAdapterError(err, targetField) {
|
|
5019
|
+
if (err instanceof NarsilError) return err;
|
|
5020
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
5021
|
+
return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
|
|
5022
|
+
field: targetField,
|
|
5023
|
+
cause: message
|
|
5024
|
+
});
|
|
5025
|
+
}
|
|
5026
|
+
async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
5027
|
+
const embeddedFields = /* @__PURE__ */ new Set();
|
|
5028
|
+
try {
|
|
5029
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
5030
|
+
const existing = resolveFieldValue(document, targetField);
|
|
5031
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
5032
|
+
const text = collectSourceText(document, sourceFields, targetField);
|
|
5033
|
+
let vector;
|
|
5034
|
+
try {
|
|
5035
|
+
const raw = await adapter.embed(text, "document", signal);
|
|
5036
|
+
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
5037
|
+
} catch (err) {
|
|
5038
|
+
throw wrapAdapterError(err, targetField);
|
|
5039
|
+
}
|
|
5040
|
+
setFieldValue(document, targetField, vector);
|
|
5041
|
+
embeddedFields.add(targetField);
|
|
5042
|
+
}
|
|
5043
|
+
} catch (err) {
|
|
5044
|
+
for (const field of embeddedFields) {
|
|
5045
|
+
deleteFieldValue(document, field);
|
|
5046
|
+
}
|
|
5047
|
+
throw err;
|
|
5048
|
+
}
|
|
5049
|
+
return embeddedFields;
|
|
5050
|
+
}
|
|
5051
|
+
var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([
|
|
5052
|
+
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
5053
|
+
ErrorCodes.DOC_VALIDATION_FAILED
|
|
5054
|
+
]);
|
|
5055
|
+
function isDocumentSourceError(err) {
|
|
5056
|
+
return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
|
|
5057
|
+
}
|
|
5058
|
+
async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
|
|
5059
|
+
const embedded = /* @__PURE__ */ new Map();
|
|
5060
|
+
const failed = /* @__PURE__ */ new Map();
|
|
5061
|
+
try {
|
|
5062
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
5063
|
+
const needsEmbedding = [];
|
|
5064
|
+
for (let i = 0; i < documents.length; i++) {
|
|
5065
|
+
if (failed.has(i)) continue;
|
|
5066
|
+
const doc = documents[i];
|
|
5067
|
+
const existing = resolveFieldValue(doc, targetField);
|
|
5068
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
5069
|
+
try {
|
|
5070
|
+
const text = collectSourceText(doc, sourceFields, targetField, i);
|
|
5071
|
+
needsEmbedding.push({ docIndex: i, text });
|
|
5072
|
+
} catch (err) {
|
|
5073
|
+
if (isDocumentSourceError(err)) {
|
|
5074
|
+
failed.set(i, err);
|
|
5075
|
+
} else {
|
|
5076
|
+
throw err;
|
|
5077
|
+
}
|
|
5078
|
+
}
|
|
5079
|
+
}
|
|
5080
|
+
if (needsEmbedding.length === 0) continue;
|
|
5081
|
+
const texts = needsEmbedding.map((entry) => entry.text);
|
|
5082
|
+
let vectors;
|
|
5083
|
+
try {
|
|
5084
|
+
if (adapter.embedBatch) {
|
|
5085
|
+
vectors = await adapter.embedBatch(texts, "document", signal);
|
|
5086
|
+
} else {
|
|
5087
|
+
vectors = [];
|
|
5088
|
+
const concurrency = 8;
|
|
5089
|
+
for (let start = 0; start < texts.length; start += concurrency) {
|
|
5090
|
+
const chunk = texts.slice(start, start + concurrency);
|
|
5091
|
+
const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
|
|
5092
|
+
vectors.push(...chunkResults);
|
|
5093
|
+
}
|
|
5094
|
+
}
|
|
5095
|
+
} catch (err) {
|
|
5096
|
+
throw wrapAdapterError(err, targetField);
|
|
5097
|
+
}
|
|
5098
|
+
if (vectors.length !== needsEmbedding.length) {
|
|
5099
|
+
throw new NarsilError(
|
|
5100
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
5101
|
+
`Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
|
|
5102
|
+
{ field: targetField, expected: needsEmbedding.length, actual: vectors.length }
|
|
5103
|
+
);
|
|
5104
|
+
}
|
|
5105
|
+
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
5106
|
+
const { docIndex } = needsEmbedding[j];
|
|
5107
|
+
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
5108
|
+
setFieldValue(documents[docIndex], targetField, validated);
|
|
5109
|
+
let fieldSet = embedded.get(docIndex);
|
|
5110
|
+
if (!fieldSet) {
|
|
5111
|
+
fieldSet = /* @__PURE__ */ new Set();
|
|
5112
|
+
embedded.set(docIndex, fieldSet);
|
|
5113
|
+
}
|
|
5114
|
+
fieldSet.add(targetField);
|
|
5115
|
+
}
|
|
5116
|
+
}
|
|
5117
|
+
} catch (err) {
|
|
5118
|
+
for (const [docIndex, fields] of embedded) {
|
|
5119
|
+
for (const field of fields) {
|
|
5120
|
+
deleteFieldValue(documents[docIndex], field);
|
|
5121
|
+
}
|
|
5122
|
+
}
|
|
5123
|
+
throw err;
|
|
5124
|
+
}
|
|
5125
|
+
return { embedded, failed };
|
|
5126
|
+
}
|
|
5127
|
+
|
|
4978
5128
|
// src/engine/mutations/durable-rollback.ts
|
|
4979
5129
|
function rollbackFailed(docId, originalError, rollbackError) {
|
|
4980
5130
|
throw new NarsilError(
|
|
@@ -4992,18 +5142,24 @@ async function rollbackInsertedDocument(ctx, indexName, docId, inserted, origina
|
|
|
4992
5142
|
return;
|
|
4993
5143
|
}
|
|
4994
5144
|
try {
|
|
4995
|
-
await
|
|
4996
|
-
|
|
5145
|
+
await changeIndexOutsideExternalWaits(async () => {
|
|
5146
|
+
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
|
|
5147
|
+
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
5148
|
+
});
|
|
4997
5149
|
} catch (rollbackError) {
|
|
4998
5150
|
rollbackFailed(docId, originalError, rollbackError);
|
|
4999
5151
|
}
|
|
5000
5152
|
}
|
|
5001
5153
|
async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, originalError) {
|
|
5002
|
-
if (restoreDoc === void 0
|
|
5154
|
+
if (restoreDoc === void 0) {
|
|
5003
5155
|
return;
|
|
5004
5156
|
}
|
|
5157
|
+
const manager = ctx.requireManager(indexName);
|
|
5005
5158
|
try {
|
|
5006
|
-
await
|
|
5159
|
+
await changeIndexOutsideExternalWaits(async () => {
|
|
5160
|
+
if (manager.has(docId)) return;
|
|
5161
|
+
await ctx.executor.execute({ type: "insert", indexName, docId, document: restoreDoc, requestId: docId });
|
|
5162
|
+
});
|
|
5007
5163
|
} catch (rollbackError) {
|
|
5008
5164
|
rollbackFailed(docId, originalError, rollbackError);
|
|
5009
5165
|
}
|
|
@@ -5013,12 +5169,149 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
5013
5169
|
return;
|
|
5014
5170
|
}
|
|
5015
5171
|
try {
|
|
5016
|
-
await
|
|
5172
|
+
await changeIndexOutsideExternalWaits(async () => {
|
|
5173
|
+
await ctx.executor.execute({ type: "update", indexName, docId, document: restoreDoc, requestId: docId });
|
|
5174
|
+
});
|
|
5017
5175
|
} catch (rollbackError) {
|
|
5018
5176
|
rollbackFailed(docId, originalError, rollbackError);
|
|
5019
5177
|
}
|
|
5020
5178
|
}
|
|
5021
5179
|
|
|
5180
|
+
// src/engine/mutations/insert-admission.ts
|
|
5181
|
+
function providedDocId(document) {
|
|
5182
|
+
if (document === null || typeof document !== "object") return void 0;
|
|
5183
|
+
const id = document.id;
|
|
5184
|
+
return typeof id === "string" && id.length > 0 ? id : void 0;
|
|
5185
|
+
}
|
|
5186
|
+
function requireDocumentObject(document) {
|
|
5187
|
+
if (document !== null && typeof document === "object" && !Array.isArray(document)) {
|
|
5188
|
+
requireDocumentWithinSizeLimit(document);
|
|
5189
|
+
return;
|
|
5190
|
+
}
|
|
5191
|
+
const received = Array.isArray(document) ? "a list" : document === null ? "null" : `a ${typeof document}`;
|
|
5192
|
+
throw new NarsilError(
|
|
5193
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5194
|
+
`Each document must be an object, while this entry is ${received}`,
|
|
5195
|
+
{
|
|
5196
|
+
received
|
|
5197
|
+
}
|
|
5198
|
+
);
|
|
5199
|
+
}
|
|
5200
|
+
function requireMatchingDocId(docId, document) {
|
|
5201
|
+
const embeddedDocId = providedDocId(document);
|
|
5202
|
+
if (embeddedDocId === void 0 || embeddedDocId === docId) return;
|
|
5203
|
+
throw new NarsilError(
|
|
5204
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5205
|
+
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
5206
|
+
{ docId, documentId: embeddedDocId }
|
|
5207
|
+
);
|
|
5208
|
+
}
|
|
5209
|
+
function asBatchInsertError(err) {
|
|
5210
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
5211
|
+
}
|
|
5212
|
+
function createPendingAdmissions() {
|
|
5213
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
5214
|
+
}
|
|
5215
|
+
function capsEachPartition(ctx, manager) {
|
|
5216
|
+
return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
|
|
5217
|
+
}
|
|
5218
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
5219
|
+
pending.total += 1;
|
|
5220
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
5221
|
+
const partitionId = manager.routePartition(docId);
|
|
5222
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
5223
|
+
}
|
|
5224
|
+
function admitInsert(ctx, indexName, manager, docId, pending) {
|
|
5225
|
+
const rebalancing = ctx.isRebalancing(indexName);
|
|
5226
|
+
if (rebalancing) {
|
|
5227
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
5228
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
5229
|
+
if (exists) {
|
|
5230
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
5231
|
+
}
|
|
5232
|
+
}
|
|
5233
|
+
assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing);
|
|
5234
|
+
}
|
|
5235
|
+
function admitBufferedUpsert(ctx, indexName, manager, docId) {
|
|
5236
|
+
assertRoomFor(ctx, indexName, manager, docId, void 0, true);
|
|
5237
|
+
}
|
|
5238
|
+
function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
|
|
5239
|
+
if (ctx.capacityScope === "partition") {
|
|
5240
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
5241
|
+
const partitionId = manager.routePartition(docId);
|
|
5242
|
+
manager.assertPartitionCapacity(partitionId, pending?.byPartition.get(partitionId) ?? 0);
|
|
5243
|
+
return;
|
|
5244
|
+
}
|
|
5245
|
+
const pendingAdmitted = pending?.total ?? 0;
|
|
5246
|
+
if (!rebalancing) {
|
|
5247
|
+
manager.assertCapacity(pendingAdmitted);
|
|
5248
|
+
return;
|
|
5249
|
+
}
|
|
5250
|
+
manager.assertCapacity(
|
|
5251
|
+
ctx.pendingRebalanceWrites(indexName) + pendingAdmitted,
|
|
5252
|
+
ctx.rebalanceTargetPartitionCount(indexName)
|
|
5253
|
+
);
|
|
5254
|
+
}
|
|
5255
|
+
function collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
|
|
5256
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5257
|
+
try {
|
|
5258
|
+
validateRequiredFields(documents[i], required);
|
|
5259
|
+
} catch (err) {
|
|
5260
|
+
chunkFailedIndexes.add(i);
|
|
5261
|
+
failed.push({ docId: providedDocId(documents[i]) ?? "", error: asBatchInsertError(err) });
|
|
5262
|
+
}
|
|
5263
|
+
}
|
|
5264
|
+
}
|
|
5265
|
+
async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
|
|
5266
|
+
if (entry.config.embedding === void 0) return;
|
|
5267
|
+
if (entry.embeddingAdapter === null) {
|
|
5268
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5269
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
5270
|
+
try {
|
|
5271
|
+
assertDocumentCarriesMappedVectors(
|
|
5272
|
+
documents[i],
|
|
5273
|
+
entry.config.embedding,
|
|
5274
|
+
entry.embeddingAdapterName
|
|
5275
|
+
);
|
|
5276
|
+
} catch (err) {
|
|
5277
|
+
chunkFailedIndexes.add(i);
|
|
5278
|
+
failed.push({
|
|
5279
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
5280
|
+
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
5281
|
+
});
|
|
5282
|
+
}
|
|
5283
|
+
}
|
|
5284
|
+
return;
|
|
5285
|
+
}
|
|
5286
|
+
const embeddableSlice = [];
|
|
5287
|
+
const embeddableOriginalIndexes = [];
|
|
5288
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5289
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
5290
|
+
embeddableSlice.push(documents[i]);
|
|
5291
|
+
embeddableOriginalIndexes.push(i);
|
|
5292
|
+
}
|
|
5293
|
+
if (embeddableSlice.length === 0) return;
|
|
5294
|
+
try {
|
|
5295
|
+
const embedResult = await embedBatchDocumentFields(
|
|
5296
|
+
embeddableSlice,
|
|
5297
|
+
entry.config.embedding,
|
|
5298
|
+
entry.embeddingAdapter,
|
|
5299
|
+
signal
|
|
5300
|
+
);
|
|
5301
|
+
for (const [sliceIndex, error] of embedResult.failed) {
|
|
5302
|
+
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
5303
|
+
chunkFailedIndexes.add(originalIdx);
|
|
5304
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
|
|
5305
|
+
}
|
|
5306
|
+
} catch (err) {
|
|
5307
|
+
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
5308
|
+
for (const originalIdx of embeddableOriginalIndexes) {
|
|
5309
|
+
chunkFailedIndexes.add(originalIdx);
|
|
5310
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
|
|
5311
|
+
}
|
|
5312
|
+
}
|
|
5313
|
+
}
|
|
5314
|
+
|
|
5022
5315
|
// src/engine/mutations/insert.ts
|
|
5023
5316
|
async function insertDocument(ctx, indexName, document, docId, options) {
|
|
5024
5317
|
ctx.guardShutdown();
|
|
@@ -5059,7 +5352,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
5059
5352
|
}
|
|
5060
5353
|
let inserted = false;
|
|
5061
5354
|
let buffered = false;
|
|
5062
|
-
const
|
|
5355
|
+
const insertIntoIndex = async () => {
|
|
5063
5356
|
admitInsert(ctx, indexName, insertManager, resolvedDocId);
|
|
5064
5357
|
if (ctx.bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
|
|
5065
5358
|
buffered = true;
|
|
@@ -5094,6 +5387,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
5094
5387
|
throw err;
|
|
5095
5388
|
}
|
|
5096
5389
|
};
|
|
5390
|
+
const applyInsert = indexChangeOutsideExternalWaits(insertIntoIndex);
|
|
5097
5391
|
if (ctx.durability) {
|
|
5098
5392
|
try {
|
|
5099
5393
|
await ctx.durability.recordInsertOrUpdate(indexName, resolvedDocId, document, applyInsert);
|
|
@@ -5148,6 +5442,7 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
5148
5442
|
}
|
|
5149
5443
|
const failures = [];
|
|
5150
5444
|
for (const apply of applies) {
|
|
5445
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5151
5446
|
try {
|
|
5152
5447
|
await apply();
|
|
5153
5448
|
failures.push(null);
|
|
@@ -5161,7 +5456,7 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
5161
5456
|
// src/engine/mutations/insert-batch-apply.ts
|
|
5162
5457
|
function applyOf(ctx, indexName, doc, options, progress) {
|
|
5163
5458
|
const manager = ctx.requireManager(indexName);
|
|
5164
|
-
|
|
5459
|
+
const insertIntoIndex = async () => {
|
|
5165
5460
|
if (ctx.abortController.signal.aborted) {
|
|
5166
5461
|
progress.skipped = true;
|
|
5167
5462
|
throw ctx.abortController.signal.reason;
|
|
@@ -5200,6 +5495,7 @@ function applyOf(ctx, indexName, doc, options, progress) {
|
|
|
5200
5495
|
throw vecErr;
|
|
5201
5496
|
}
|
|
5202
5497
|
};
|
|
5498
|
+
return indexChangeOutsideExternalWaits(insertIntoIndex);
|
|
5203
5499
|
}
|
|
5204
5500
|
async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
5205
5501
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
@@ -5485,7 +5781,7 @@ function attachOnce(manager, attachment) {
|
|
|
5485
5781
|
}
|
|
5486
5782
|
function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
5487
5783
|
const manager = ctx.requireManager(indexName);
|
|
5488
|
-
|
|
5784
|
+
const attachToIndex = async () => {
|
|
5489
5785
|
attachOnce(manager, attachment);
|
|
5490
5786
|
progress.applied = true;
|
|
5491
5787
|
try {
|
|
@@ -5508,6 +5804,7 @@ function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
|
5508
5804
|
throw vecErr;
|
|
5509
5805
|
}
|
|
5510
5806
|
};
|
|
5807
|
+
return indexChangeOutsideExternalWaits(attachToIndex);
|
|
5511
5808
|
}
|
|
5512
5809
|
function chunksOfWholeSegments(built, frozen, memberIndexes, admitted) {
|
|
5513
5810
|
const chunks = [];
|
|
@@ -5693,6 +5990,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
|
5693
5990
|
async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
5694
5991
|
ctx.guardShutdown();
|
|
5695
5992
|
const entry = ctx.requireIndex(indexName);
|
|
5993
|
+
validateBatchList(documents, "documents");
|
|
5696
5994
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
|
|
5697
5995
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
5698
5996
|
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
@@ -5757,6 +6055,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
5757
6055
|
}
|
|
5758
6056
|
const prepared = [];
|
|
5759
6057
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
6058
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5760
6059
|
if (ctx.abortController.signal.aborted) break;
|
|
5761
6060
|
if (chunkFailedIndexes.has(i)) continue;
|
|
5762
6061
|
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
@@ -5800,6 +6099,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
5800
6099
|
const replicatedAsSegments = copiesBuildSegments && await replicateAsSegments(ctx, indexName, replicableIds, replicableDocs, options?.skipClone);
|
|
5801
6100
|
if (!replicatedAsSegments) {
|
|
5802
6101
|
for (let i = 0; i < replicableIds.length; i++) {
|
|
6102
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5803
6103
|
await ctx.orchestrator.replicateToWorkers({
|
|
5804
6104
|
type: "insert",
|
|
5805
6105
|
indexName,
|
|
@@ -5847,7 +6147,7 @@ async function prepareRemoval(ctx, indexName, docId) {
|
|
|
5847
6147
|
}
|
|
5848
6148
|
function applyOfRemoval(ctx, indexName, prepared) {
|
|
5849
6149
|
const { docId, removedSpellings } = prepared;
|
|
5850
|
-
|
|
6150
|
+
const removeFromIndex = async () => {
|
|
5851
6151
|
if (ctx.isRebalancing(indexName)) {
|
|
5852
6152
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
5853
6153
|
const exists = bufferedState !== void 0 ? bufferedState === "present" : ctx.requireManager(indexName).has(docId);
|
|
@@ -5862,6 +6162,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
5862
6162
|
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId, removedSpellings });
|
|
5863
6163
|
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
5864
6164
|
};
|
|
6165
|
+
return indexChangeOutsideExternalWaits(removeFromIndex);
|
|
5865
6166
|
}
|
|
5866
6167
|
async function settleRemoval(ctx, indexName, prepared) {
|
|
5867
6168
|
const { docId, removedSpellings } = prepared;
|
|
@@ -5899,6 +6200,7 @@ async function removeDocument(ctx, indexName, docId, options) {
|
|
|
5899
6200
|
async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
5900
6201
|
ctx.guardShutdown();
|
|
5901
6202
|
ctx.requireIndex(indexName);
|
|
6203
|
+
validateBatchList(docIds, "document IDs");
|
|
5902
6204
|
const succeeded = [];
|
|
5903
6205
|
const failed = [];
|
|
5904
6206
|
const manager = ctx.requireManager(indexName);
|
|
@@ -5908,6 +6210,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
5908
6210
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
5909
6211
|
const prepared = [];
|
|
5910
6212
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
6213
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5911
6214
|
try {
|
|
5912
6215
|
prepared.push(await prepareRemoval(ctx, indexName, docIds[i]));
|
|
5913
6216
|
} catch (err) {
|
|
@@ -6052,7 +6355,7 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
|
6052
6355
|
function applyOfUpdate(ctx, indexName, prepared) {
|
|
6053
6356
|
const manager = ctx.requireManager(indexName);
|
|
6054
6357
|
const { docId, document, partitionDoc, rollbackDoc, extractedVectors, removedSpellings } = prepared;
|
|
6055
|
-
|
|
6358
|
+
const updateInIndex = async () => {
|
|
6056
6359
|
if (ctx.isRebalancing(indexName)) {
|
|
6057
6360
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
6058
6361
|
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
@@ -6088,6 +6391,7 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
6088
6391
|
throw err;
|
|
6089
6392
|
}
|
|
6090
6393
|
};
|
|
6394
|
+
return indexChangeOutsideExternalWaits(updateInIndex);
|
|
6091
6395
|
}
|
|
6092
6396
|
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
6093
6397
|
ctx.guardShutdown();
|
|
@@ -6127,6 +6431,7 @@ function requireUpdateEntry(update) {
|
|
|
6127
6431
|
async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
6128
6432
|
ctx.guardShutdown();
|
|
6129
6433
|
const entry = ctx.requireIndex(indexName);
|
|
6434
|
+
validateBatchList(updates, "updates");
|
|
6130
6435
|
const succeeded = [];
|
|
6131
6436
|
const failed = [];
|
|
6132
6437
|
const updateBatchManager = ctx.requireManager(indexName);
|
|
@@ -6136,6 +6441,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
6136
6441
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
6137
6442
|
const prepared = [];
|
|
6138
6443
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
6444
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
6139
6445
|
const update = updates[i];
|
|
6140
6446
|
const docId = reportedDocIdOf(update);
|
|
6141
6447
|
try {
|
|
@@ -6179,51 +6485,130 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
6179
6485
|
return { succeeded, failed };
|
|
6180
6486
|
}
|
|
6181
6487
|
|
|
6182
|
-
// src/engine/
|
|
6183
|
-
|
|
6184
|
-
|
|
6185
|
-
|
|
6186
|
-
|
|
6187
|
-
if (
|
|
6188
|
-
|
|
6189
|
-
|
|
6190
|
-
|
|
6191
|
-
|
|
6488
|
+
// src/engine/memory/guarded-writes.ts
|
|
6489
|
+
var EMPTY_DOCUMENT = {};
|
|
6490
|
+
function blockFor(ctx, indexName) {
|
|
6491
|
+
const block = ctx.memoryWriteBlock;
|
|
6492
|
+
if (block === void 0) return void 0;
|
|
6493
|
+
if (block.grouping !== void 0) {
|
|
6494
|
+
ctx.guardShutdown();
|
|
6495
|
+
ctx.requireIndex(indexName);
|
|
6496
|
+
block.refuseWhileFull(indexName);
|
|
6497
|
+
return block;
|
|
6498
|
+
}
|
|
6499
|
+
try {
|
|
6500
|
+
block.refuseWhileFull(indexName);
|
|
6501
|
+
} catch (refusal) {
|
|
6502
|
+
ctx.guardShutdown();
|
|
6503
|
+
ctx.requireIndex(indexName);
|
|
6504
|
+
throw refusal;
|
|
6192
6505
|
}
|
|
6193
|
-
|
|
6194
|
-
single.set(fieldName, idx);
|
|
6195
|
-
return single;
|
|
6506
|
+
return block;
|
|
6196
6507
|
}
|
|
6197
|
-
function
|
|
6198
|
-
|
|
6199
|
-
|
|
6200
|
-
|
|
6508
|
+
async function withReservation(grouping, indexName, document, write) {
|
|
6509
|
+
if (grouping === void 0) return write();
|
|
6510
|
+
const reservation = grouping.reserveFor(indexName, document);
|
|
6511
|
+
try {
|
|
6512
|
+
return await write();
|
|
6513
|
+
} finally {
|
|
6514
|
+
reservation.release();
|
|
6201
6515
|
}
|
|
6202
6516
|
}
|
|
6203
|
-
async function
|
|
6204
|
-
const
|
|
6205
|
-
|
|
6206
|
-
|
|
6517
|
+
async function writeInGroups(grouping, indexName, items, write) {
|
|
6518
|
+
const succeeded = [];
|
|
6519
|
+
const failed = [];
|
|
6520
|
+
await grouping.applyInGroups(indexName, {
|
|
6521
|
+
items,
|
|
6522
|
+
failed,
|
|
6523
|
+
apply: async (group) => {
|
|
6524
|
+
const result = await write(group);
|
|
6525
|
+
for (const docId of result.succeeded) succeeded.push(docId);
|
|
6526
|
+
for (const failure of result.failed) failed.push(failure);
|
|
6527
|
+
}
|
|
6528
|
+
});
|
|
6529
|
+
return { succeeded, failed };
|
|
6530
|
+
}
|
|
6531
|
+
function updateItemOf(update) {
|
|
6532
|
+
const entry = update;
|
|
6533
|
+
if (entry === null || typeof entry !== "object") return { docId: "", document: EMPTY_DOCUMENT, update };
|
|
6534
|
+
const { docId, document } = entry;
|
|
6535
|
+
return {
|
|
6536
|
+
docId: typeof docId === "string" ? docId : "",
|
|
6537
|
+
document: document !== null && typeof document === "object" ? document : EMPTY_DOCUMENT,
|
|
6538
|
+
update
|
|
6539
|
+
};
|
|
6540
|
+
}
|
|
6541
|
+
async function insertDocument2(ctx, indexName, document, docId, options) {
|
|
6542
|
+
const block = blockFor(ctx, indexName);
|
|
6543
|
+
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options);
|
|
6544
|
+
return withReservation(
|
|
6545
|
+
block.grouping,
|
|
6546
|
+
indexName,
|
|
6547
|
+
document,
|
|
6548
|
+
() => insertDocument(ctx, indexName, document, docId, options)
|
|
6549
|
+
);
|
|
6550
|
+
}
|
|
6551
|
+
async function insertDocumentBatch2(ctx, indexName, documents, options) {
|
|
6552
|
+
const block = blockFor(ctx, indexName);
|
|
6553
|
+
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents, options);
|
|
6554
|
+
const grouping = block.grouping;
|
|
6555
|
+
if (grouping === void 0 || !Array.isArray(documents) || documents.length === 0) {
|
|
6556
|
+
return insertDocumentBatch(ctx, indexName, documents, options);
|
|
6207
6557
|
}
|
|
6558
|
+
const items = documents.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
6559
|
+
return writeInGroups(
|
|
6560
|
+
grouping,
|
|
6561
|
+
indexName,
|
|
6562
|
+
items,
|
|
6563
|
+
(group) => insertDocumentBatch(
|
|
6564
|
+
ctx,
|
|
6565
|
+
indexName,
|
|
6566
|
+
group.map((item) => item.document),
|
|
6567
|
+
options
|
|
6568
|
+
)
|
|
6569
|
+
);
|
|
6208
6570
|
}
|
|
6209
|
-
function
|
|
6210
|
-
const
|
|
6211
|
-
|
|
6212
|
-
|
|
6213
|
-
|
|
6214
|
-
|
|
6215
|
-
|
|
6216
|
-
|
|
6217
|
-
|
|
6218
|
-
|
|
6219
|
-
|
|
6220
|
-
|
|
6221
|
-
|
|
6222
|
-
|
|
6223
|
-
|
|
6224
|
-
|
|
6571
|
+
async function updateDocument2(ctx, indexName, docId, document, options) {
|
|
6572
|
+
const block = blockFor(ctx, indexName);
|
|
6573
|
+
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options);
|
|
6574
|
+
return withReservation(
|
|
6575
|
+
block.grouping,
|
|
6576
|
+
indexName,
|
|
6577
|
+
document,
|
|
6578
|
+
() => updateDocument(ctx, indexName, docId, document, options)
|
|
6579
|
+
);
|
|
6580
|
+
}
|
|
6581
|
+
async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
6582
|
+
const block = blockFor(ctx, indexName);
|
|
6583
|
+
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options);
|
|
6584
|
+
const grouping = block.grouping;
|
|
6585
|
+
if (grouping === void 0 || !Array.isArray(updates) || updates.length === 0) {
|
|
6586
|
+
return updateDocumentBatch(ctx, indexName, updates, options);
|
|
6225
6587
|
}
|
|
6226
|
-
return
|
|
6588
|
+
return writeInGroups(
|
|
6589
|
+
grouping,
|
|
6590
|
+
indexName,
|
|
6591
|
+
updates.map(updateItemOf),
|
|
6592
|
+
(group) => updateDocumentBatch(
|
|
6593
|
+
ctx,
|
|
6594
|
+
indexName,
|
|
6595
|
+
group.map((item) => item.update),
|
|
6596
|
+
options
|
|
6597
|
+
)
|
|
6598
|
+
);
|
|
6599
|
+
}
|
|
6600
|
+
async function removeDocument2(ctx, indexName, docId, options) {
|
|
6601
|
+
await removeDocument(ctx, indexName, docId, options);
|
|
6602
|
+
ctx.memoryWriteBlock?.memoryReleased();
|
|
6603
|
+
}
|
|
6604
|
+
async function removeDocumentBatch2(ctx, indexName, docIds, options) {
|
|
6605
|
+
const result = await removeDocumentBatch(ctx, indexName, docIds, options);
|
|
6606
|
+
if (result.succeeded.length > 0) ctx.memoryWriteBlock?.memoryReleased();
|
|
6607
|
+
return result;
|
|
6608
|
+
}
|
|
6609
|
+
async function dropEngineIndex2(core, name) {
|
|
6610
|
+
await dropEngineIndex(core, name);
|
|
6611
|
+
core.mutationCtx.memoryWriteBlock?.memoryReleased();
|
|
6227
6612
|
}
|
|
6228
6613
|
|
|
6229
6614
|
// src/runtime/process-memory.ts
|
|
@@ -6302,6 +6687,24 @@ function bm25Operations(core) {
|
|
|
6302
6687
|
};
|
|
6303
6688
|
}
|
|
6304
6689
|
|
|
6690
|
+
// src/narsil/internals.ts
|
|
6691
|
+
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
6692
|
+
var rerankWindowSources = /* @__PURE__ */ new WeakMap();
|
|
6693
|
+
function bindEngineCore(engine, core) {
|
|
6694
|
+
engineCores.set(engine, core);
|
|
6695
|
+
}
|
|
6696
|
+
function engineCoreOf(engine) {
|
|
6697
|
+
return engineCores.get(engine);
|
|
6698
|
+
}
|
|
6699
|
+
function bindRerankWindowSource(owner, read) {
|
|
6700
|
+
rerankWindowSources.set(owner, read);
|
|
6701
|
+
}
|
|
6702
|
+
function rerankWindowCapOf(owner) {
|
|
6703
|
+
const core = engineCores.get(owner);
|
|
6704
|
+
if (core !== void 0) return core.rerankSettings.maxWindow;
|
|
6705
|
+
return rerankWindowSources.get(owner)?.();
|
|
6706
|
+
}
|
|
6707
|
+
|
|
6305
6708
|
// src/narsil/portability.ts
|
|
6306
6709
|
function portabilityOperations(core, config, engine) {
|
|
6307
6710
|
async function withOpenIndex2(indexName, action) {
|
|
@@ -6333,14 +6736,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6333
6736
|
return {
|
|
6334
6737
|
snapshot(indexName) {
|
|
6335
6738
|
return withOpenIndex2(indexName, async () => {
|
|
6336
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6739
|
+
const { createSnapshot } = await import('./snapshot-EHXTOTW5.mjs');
|
|
6337
6740
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6338
6741
|
});
|
|
6339
6742
|
},
|
|
6340
6743
|
async restore(indexName, data) {
|
|
6341
6744
|
core.guardShutdown();
|
|
6342
6745
|
validateIndexName(indexName);
|
|
6343
|
-
const restored = import('./restore-queue-
|
|
6746
|
+
const restored = import('./restore-queue-BQWWNNQI.mjs').then(
|
|
6344
6747
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6345
6748
|
);
|
|
6346
6749
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6349,8 +6752,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6349
6752
|
exportDocuments(indexName) {
|
|
6350
6753
|
return withOpenIndex2(indexName, async () => {
|
|
6351
6754
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6352
|
-
import('./snapshot-
|
|
6353
|
-
import('./write-
|
|
6755
|
+
import('./snapshot-EHXTOTW5.mjs'),
|
|
6756
|
+
import('./write-SDJXXQ2G.mjs')
|
|
6354
6757
|
]);
|
|
6355
6758
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6356
6759
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6358,7 +6761,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6358
6761
|
},
|
|
6359
6762
|
importDocumentExport(indexName, data) {
|
|
6360
6763
|
core.guardShutdown();
|
|
6361
|
-
const imported = import('./import-
|
|
6764
|
+
const imported = import('./import-DX55PN4G.mjs').then(
|
|
6362
6765
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
6363
6766
|
);
|
|
6364
6767
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -6366,6 +6769,58 @@ function portabilityOperations(core, config, engine) {
|
|
|
6366
6769
|
};
|
|
6367
6770
|
}
|
|
6368
6771
|
|
|
6772
|
+
// src/engine/resolve-vector-text.ts
|
|
6773
|
+
async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
|
|
6774
|
+
if (!params.vector) return params;
|
|
6775
|
+
if (params.vector.text === void 0 && params.vector.value === void 0) return params;
|
|
6776
|
+
if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
|
|
6777
|
+
if (params.vector.text !== void 0 && params.vector.value !== void 0) {
|
|
6778
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
|
|
6779
|
+
}
|
|
6780
|
+
if (!embeddingAdapter) {
|
|
6781
|
+
throw new NarsilError(
|
|
6782
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6783
|
+
embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
|
|
6784
|
+
embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
|
|
6785
|
+
);
|
|
6786
|
+
}
|
|
6787
|
+
if (typeof params.vector.text !== "string") {
|
|
6788
|
+
throw new NarsilError(
|
|
6789
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
6790
|
+
`Vector query 'text' must be a string, got ${typeof params.vector.text}`
|
|
6791
|
+
);
|
|
6792
|
+
}
|
|
6793
|
+
if (params.vector.text.length === 0) {
|
|
6794
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
|
|
6795
|
+
}
|
|
6796
|
+
let raw;
|
|
6797
|
+
try {
|
|
6798
|
+
raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
|
|
6799
|
+
} catch (err) {
|
|
6800
|
+
if (err instanceof NarsilError) throw err;
|
|
6801
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
6802
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, `The embedding adapter cannot embed the query: ${message}`, {
|
|
6803
|
+
cause: message
|
|
6804
|
+
});
|
|
6805
|
+
}
|
|
6806
|
+
if (!(raw instanceof Float32Array)) {
|
|
6807
|
+
throw new NarsilError(
|
|
6808
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
6809
|
+
`Adapter returned ${typeof raw} for query embedding, expected Float32Array`
|
|
6810
|
+
);
|
|
6811
|
+
}
|
|
6812
|
+
if (raw.length !== embeddingAdapter.dimensions) {
|
|
6813
|
+
throw new NarsilError(
|
|
6814
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
6815
|
+
`Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
|
|
6816
|
+
{ expected: embeddingAdapter.dimensions, actual: raw.length }
|
|
6817
|
+
);
|
|
6818
|
+
}
|
|
6819
|
+
const resolved = { ...params.vector, value: Array.from(raw) };
|
|
6820
|
+
delete resolved.text;
|
|
6821
|
+
return { ...params, vector: resolved };
|
|
6822
|
+
}
|
|
6823
|
+
|
|
6369
6824
|
// src/narsil/reads.ts
|
|
6370
6825
|
async function enterRead(core, indexName) {
|
|
6371
6826
|
core.guardShutdown();
|
|
@@ -6417,7 +6872,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
6417
6872
|
const manager = core.requireManager(indexName);
|
|
6418
6873
|
const scoped = scopedParams(params, options);
|
|
6419
6874
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
6420
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
6875
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-L2DZYY6N.mjs');
|
|
6421
6876
|
const resolvedParams = await resolveVectorText(
|
|
6422
6877
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
6423
6878
|
entry.embeddingAdapter,
|
|
@@ -6613,6 +7068,9 @@ function createNarsilFromCore(core, config) {
|
|
|
6613
7068
|
const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
|
|
6614
7069
|
return emitWriteEventAfter(core, indexName, written);
|
|
6615
7070
|
}
|
|
7071
|
+
function withCountedWrite(indexName, action) {
|
|
7072
|
+
return countWrite(() => withOpenIndexWrite(indexName, action));
|
|
7073
|
+
}
|
|
6616
7074
|
const narsil = {
|
|
6617
7075
|
createIndex(name, indexConfig) {
|
|
6618
7076
|
return emitWriteEventAfter(
|
|
@@ -6628,7 +7086,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6628
7086
|
registerEngineRerankAdapter(core, name, adapter);
|
|
6629
7087
|
},
|
|
6630
7088
|
dropIndex(name) {
|
|
6631
|
-
return emitWriteEventAfter(core, name,
|
|
7089
|
+
return emitWriteEventAfter(core, name, dropEngineIndex2(core, name));
|
|
6632
7090
|
},
|
|
6633
7091
|
open(indexName) {
|
|
6634
7092
|
guardShutdown();
|
|
@@ -6674,22 +7132,22 @@ function createNarsilFromCore(core, config) {
|
|
|
6674
7132
|
return executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
6675
7133
|
},
|
|
6676
7134
|
insert(indexName, document, docId, options) {
|
|
6677
|
-
return
|
|
7135
|
+
return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
|
|
6678
7136
|
},
|
|
6679
7137
|
insertBatch(indexName, documents, options) {
|
|
6680
|
-
return
|
|
7138
|
+
return withCountedWrite(indexName, () => insertDocumentBatch2(mutationCtx, indexName, documents, options));
|
|
6681
7139
|
},
|
|
6682
7140
|
remove(indexName, docId, options) {
|
|
6683
|
-
return
|
|
7141
|
+
return withCountedWrite(indexName, () => removeDocument2(mutationCtx, indexName, docId, options));
|
|
6684
7142
|
},
|
|
6685
7143
|
removeBatch(indexName, docIds, options) {
|
|
6686
|
-
return
|
|
7144
|
+
return withCountedWrite(indexName, () => removeDocumentBatch2(mutationCtx, indexName, docIds, options));
|
|
6687
7145
|
},
|
|
6688
7146
|
update(indexName, docId, document, options) {
|
|
6689
|
-
return
|
|
7147
|
+
return withCountedWrite(indexName, () => updateDocument2(mutationCtx, indexName, docId, document, options));
|
|
6690
7148
|
},
|
|
6691
7149
|
updateBatch(indexName, updates, options) {
|
|
6692
|
-
return
|
|
7150
|
+
return withCountedWrite(indexName, () => updateDocumentBatch2(mutationCtx, indexName, updates, options));
|
|
6693
7151
|
},
|
|
6694
7152
|
waitForWrites(indexName) {
|
|
6695
7153
|
guardShutdown();
|
|
@@ -6739,7 +7197,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6739
7197
|
const entry = requireIndex(indexName);
|
|
6740
7198
|
await withOpenIndexWrite(indexName, async () => {
|
|
6741
7199
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
6742
|
-
return (await import('./analysis-change-
|
|
7200
|
+
return (await import('./analysis-change-ZCJNYBJM.mjs')).resumeAnalysisChange(core, indexName);
|
|
6743
7201
|
});
|
|
6744
7202
|
},
|
|
6745
7203
|
async changeAnalysis(indexName, change) {
|
|
@@ -6747,7 +7205,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6747
7205
|
requireIndex(indexName);
|
|
6748
7206
|
const changed = withOpenIndex2(
|
|
6749
7207
|
indexName,
|
|
6750
|
-
async () => (await import('./analysis-change-
|
|
7208
|
+
async () => (await import('./analysis-change-ZCJNYBJM.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
6751
7209
|
);
|
|
6752
7210
|
await emitWriteEventAfter(core, indexName, changed);
|
|
6753
7211
|
},
|
|
@@ -6785,7 +7243,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6785
7243
|
}
|
|
6786
7244
|
rebalancesStarting.set(indexName, started);
|
|
6787
7245
|
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
6788
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
7246
|
+
const { executeRebalance } = await import('./analysis-change-ZCJNYBJM.mjs');
|
|
6789
7247
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
6790
7248
|
releaseWrites();
|
|
6791
7249
|
return running;
|
|
@@ -6854,6 +7312,7 @@ function createNarsilFromCore(core, config) {
|
|
|
6854
7312
|
async compactVectors(indexName, fieldName) {
|
|
6855
7313
|
guardShutdown();
|
|
6856
7314
|
requireIndex(indexName);
|
|
7315
|
+
const { compactVectors } = await loadVectorCode();
|
|
6857
7316
|
await withOpenIndexWrite(indexName, async () => {
|
|
6858
7317
|
compactVectors(requireManager(indexName), indexName, fieldName);
|
|
6859
7318
|
});
|
|
@@ -6861,13 +7320,15 @@ function createNarsilFromCore(core, config) {
|
|
|
6861
7320
|
async optimizeVectors(indexName, fieldName) {
|
|
6862
7321
|
guardShutdown();
|
|
6863
7322
|
requireIndex(indexName);
|
|
7323
|
+
const { optimizeVectors } = await loadVectorCode();
|
|
6864
7324
|
await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
|
|
6865
7325
|
},
|
|
6866
7326
|
vectorMaintenanceStatus(indexName) {
|
|
6867
7327
|
guardShutdown();
|
|
6868
7328
|
requireIndex(indexName);
|
|
6869
7329
|
const manager = executor.getManager(indexName);
|
|
6870
|
-
|
|
7330
|
+
const vectorCode = loadedVectorCode();
|
|
7331
|
+
return manager === void 0 || vectorCode === null ? [] : vectorCode.getVectorMaintenanceStatus(manager);
|
|
6871
7332
|
},
|
|
6872
7333
|
on(event, handler) {
|
|
6873
7334
|
const key = event;
|
|
@@ -6909,4 +7370,4 @@ async function createNarsil(config) {
|
|
|
6909
7370
|
// src/index.ts
|
|
6910
7371
|
var VERSION = "0.1.0";
|
|
6911
7372
|
|
|
6912
|
-
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, resolveRunnableConfig, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|
|
7373
|
+
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, bindRerankWindowSource, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createMemoryMonitor, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineCoreOf, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, memoryFractionsOf, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, providedDocId, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, rerankWindowCapOf, resolveRunnableConfig, resolveVectorText, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, toError2 as toError, validateEmbeddingConfig, validateMemoryFractions, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|