@delali/narsil 0.3.1-next.145 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
- package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
- package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
- package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
- package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
- package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
- package/dist/chunk-3MXV3VHR.mjs +46 -0
- package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
- package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
- package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
- package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
- package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
- package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
- package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/chunk-BTPGQC6Z.mjs +27 -0
- package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
- package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
- package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
- package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
- package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
- package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
- package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
- package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
- package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
- package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
- package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
- package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
- package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +2 -0
- package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
- package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
- package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +150 -255
- package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +539 -5
- package/dist/index.mjs +22 -16
- package/dist/index.react-native.mjs +19 -0
- package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
- package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
- package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
- package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
- package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/react-native/partial-words.mjs +733 -0
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +32 -28
- package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.mjs +7 -5
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
- package/package.json +88 -7
- package/dist/analysis-change-D5APH4Z3.mjs +0 -34
- package/dist/browser/import-FTMLTPGQ.mjs +0 -9
- package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
- package/dist/import-6KXFY6XI.mjs +0 -35
- package/dist/thread-OV6T555H.mjs +0 -28
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -1,24 +1,260 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-
|
|
1
|
+
import { DEFAULT_BLOCK_WRITES_ABOVE, DEFAULT_RESUME_WRITES_BELOW, validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, ENGINE_MEMORY_FIGURES_MAX_AGE_MS, storedBM25Params, readEngineMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBatchList, validateDocId } from './chunk-6F6USYII.mjs';
|
|
2
|
+
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, generateId, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-O4KUNDVF.mjs';
|
|
3
|
+
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, readHeapStatistics, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, encodeSegmentState } from './chunk-3HV7M5OJ.mjs';
|
|
4
|
+
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
2
5
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
3
|
-
import {
|
|
6
|
+
import { detectRuntime, resolveWorkerEntry, missingWorkerEntry, workerResourceLimits } from './chunk-6BWLDLS3.mjs';
|
|
7
|
+
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-XIHUI4KB.mjs';
|
|
8
|
+
import { countEngineThread } from './chunk-Q2UL2TPC.mjs';
|
|
9
|
+
import { splitWorkerBudget, resolveWorkerCount, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-3MXV3VHR.mjs';
|
|
4
10
|
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { countEngineThread } from './chunk-EBIHRL7B.mjs';
|
|
8
|
-
import { getLanguage, withOpenIndex, oneAtATime } from './chunk-FRJ5RGX7.mjs';
|
|
11
|
+
import { getLanguage, withOpenIndex, oneAtATime } from './chunk-2HJWQPR6.mjs';
|
|
12
|
+
import { loadVectorCodeFor, loadedVectorCode, loadVectorCode, requireLoadedVectorCode } from './chunk-ZAYWOHVG.mjs';
|
|
9
13
|
import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
14
|
+
import { collectAndReturnPages } from './chunk-QYRNLLLI.mjs';
|
|
15
|
+
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXNQEW43.mjs';
|
|
16
|
+
import { requireSearchBackendSettings } from './chunk-NVTFQFUL.mjs';
|
|
17
|
+
import { countWrite, turnsAreBudgeted, turnIsSpent, yieldTurn, indexChangeOutsideExternalWaits, changeIndexOutsideExternalWaits } from './chunk-LQG5G2LY.mjs';
|
|
12
18
|
import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
13
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
14
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, getNestedValue, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
15
|
-
import { isSingleStringFieldType } from './chunk-
|
|
19
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-U46IZNCW.mjs';
|
|
20
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, getNestedValue, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-SOUZMPME.mjs';
|
|
21
|
+
import { isSingleStringFieldType } from './chunk-KYVRZAYP.mjs';
|
|
16
22
|
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
17
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
23
|
+
import { ErrorCodes, NarsilError } from './chunk-MDF6PDIA.mjs';
|
|
18
24
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
19
25
|
import { encode, decode } from '@msgpack/msgpack';
|
|
20
26
|
import * as zlib from 'zlib';
|
|
21
27
|
|
|
28
|
+
// src/engine/memory/fractions.ts
|
|
29
|
+
function memoryFractionsOf(config) {
|
|
30
|
+
const block = config?.blockWritesAbove ?? DEFAULT_BLOCK_WRITES_ABOVE;
|
|
31
|
+
const resume = config?.resumeWritesBelow ?? DEFAULT_RESUME_WRITES_BELOW;
|
|
32
|
+
return {
|
|
33
|
+
processBlock: block,
|
|
34
|
+
processResume: resume,
|
|
35
|
+
heapBlock: Math.min(block, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE),
|
|
36
|
+
heapResume: Math.min(resume, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE)
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// src/engine/memory/monitor.ts
|
|
41
|
+
function fractionOf(usedBytes, limitBytes) {
|
|
42
|
+
return limitBytes > 0 ? usedBytes / limitBytes : 0;
|
|
43
|
+
}
|
|
44
|
+
function reportOf(figures, full) {
|
|
45
|
+
return {
|
|
46
|
+
processBytes: figures.processBytes,
|
|
47
|
+
processLimitBytes: figures.processLimitBytes,
|
|
48
|
+
heapBytes: figures.heapBytes,
|
|
49
|
+
heapLimitBytes: figures.heapLimitBytes,
|
|
50
|
+
full
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
function atOrAboveResume(figures, fractions) {
|
|
54
|
+
return fractionOf(figures.processBytes + figures.youngReserveBytes, figures.processLimitBytes) >= fractions.processResume || fractionOf(figures.heapBytes, figures.heapLimitBytes) >= fractions.heapResume;
|
|
55
|
+
}
|
|
56
|
+
function nextFullState(wasFull, figures, fractions) {
|
|
57
|
+
if (wasFull) return atOrAboveResume(figures, fractions);
|
|
58
|
+
return fractionOf(figures.processBytes + figures.youngReserveBytes, figures.processLimitBytes) > fractions.processBlock || fractionOf(figures.heapBytes, figures.heapLimitBytes) > fractions.heapBlock;
|
|
59
|
+
}
|
|
60
|
+
function createMemoryMonitor(options) {
|
|
61
|
+
let latest = null;
|
|
62
|
+
let timer = null;
|
|
63
|
+
let lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
64
|
+
let lastSampleAt = Number.NEGATIVE_INFINITY;
|
|
65
|
+
let collectionCount = 0;
|
|
66
|
+
let latestMeasured = null;
|
|
67
|
+
function figuresWithoutGarbageBeforeRefusal(figures) {
|
|
68
|
+
if (!options.collectGarbageBeforeRefusal()) {
|
|
69
|
+
return options.collectYoungGenerationBeforeRefusal() ? options.read() ?? figures : figures;
|
|
70
|
+
}
|
|
71
|
+
collectionCount++;
|
|
72
|
+
return options.read() ?? figures;
|
|
73
|
+
}
|
|
74
|
+
function figuresWithoutGarbage(figures) {
|
|
75
|
+
const now = Date.now();
|
|
76
|
+
if (now - lastCollectionAt < options.collectionIntervalMs) {
|
|
77
|
+
return figures;
|
|
78
|
+
}
|
|
79
|
+
lastCollectionAt = now;
|
|
80
|
+
if (!options.collectGarbage()) {
|
|
81
|
+
return figures;
|
|
82
|
+
}
|
|
83
|
+
collectionCount++;
|
|
84
|
+
return options.read() ?? figures;
|
|
85
|
+
}
|
|
86
|
+
function record(figures) {
|
|
87
|
+
const wasFull = latest?.full ?? false;
|
|
88
|
+
let full = nextFullState(wasFull, figures, options.fractions);
|
|
89
|
+
let settled2 = figures;
|
|
90
|
+
if (full) {
|
|
91
|
+
settled2 = figuresWithoutGarbage(figures);
|
|
92
|
+
full = nextFullState(wasFull, settled2, options.fractions);
|
|
93
|
+
}
|
|
94
|
+
latestMeasured = settled2;
|
|
95
|
+
latest = reportOf(settled2, full);
|
|
96
|
+
if (full !== wasFull) {
|
|
97
|
+
options.onFullChange(full);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function sample() {
|
|
101
|
+
lastSampleAt = Date.now();
|
|
102
|
+
const figures = options.read();
|
|
103
|
+
if (figures !== null) {
|
|
104
|
+
record(figures);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function latestFigures() {
|
|
108
|
+
return latestMeasured;
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
report: () => latest,
|
|
112
|
+
isFull: () => latest?.full ?? false,
|
|
113
|
+
sample,
|
|
114
|
+
measureWithoutGarbage() {
|
|
115
|
+
lastSampleAt = Date.now();
|
|
116
|
+
const figures = options.read();
|
|
117
|
+
if (figures === null) return latestFigures();
|
|
118
|
+
record(figuresWithoutGarbageBeforeRefusal(figures));
|
|
119
|
+
return latestFigures();
|
|
120
|
+
},
|
|
121
|
+
refusedAfterCollection(figures) {
|
|
122
|
+
if (latest?.full === true) return;
|
|
123
|
+
if (!atOrAboveResume(figures, options.fractions)) return;
|
|
124
|
+
latestMeasured = { ...figures };
|
|
125
|
+
latest = reportOf(figures, true);
|
|
126
|
+
options.onFullChange(true);
|
|
127
|
+
},
|
|
128
|
+
remeasureAfterRelease() {
|
|
129
|
+
lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
130
|
+
lastSampleAt = Number.NEGATIVE_INFINITY;
|
|
131
|
+
},
|
|
132
|
+
collections: () => collectionCount,
|
|
133
|
+
sampleIfOlderThan(maxAgeMs) {
|
|
134
|
+
if (timer !== null && Date.now() - lastSampleAt >= maxAgeMs) {
|
|
135
|
+
sample();
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
measureIfOlderThan(maxAgeMs) {
|
|
139
|
+
if (Date.now() - lastSampleAt >= maxAgeMs) {
|
|
140
|
+
sample();
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
start() {
|
|
144
|
+
if (timer !== null) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
sample();
|
|
148
|
+
timer = setInterval(sample, options.intervalMs);
|
|
149
|
+
timer.unref?.();
|
|
150
|
+
},
|
|
151
|
+
stop() {
|
|
152
|
+
if (timer !== null) {
|
|
153
|
+
clearInterval(timer);
|
|
154
|
+
timer = null;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// src/engine/memory/measured-block.ts
|
|
161
|
+
var MEMORY_FULL_REASON = "memory_full";
|
|
162
|
+
function finiteOrNull(value) {
|
|
163
|
+
return value !== void 0 && Number.isFinite(value) ? value : null;
|
|
164
|
+
}
|
|
165
|
+
function createMeasuredWriteBlock(settings) {
|
|
166
|
+
const fractions = memoryFractionsOf(settings);
|
|
167
|
+
const monitor = createMemoryMonitor({
|
|
168
|
+
fractions,
|
|
169
|
+
intervalMs: ENGINE_MEMORY_FIGURES_MAX_AGE_MS,
|
|
170
|
+
collectionIntervalMs: FULL_MEMORY_COLLECTION_INTERVAL_MS,
|
|
171
|
+
read: readEngineMemoryFigures,
|
|
172
|
+
collectGarbage: collectAndReturnPages,
|
|
173
|
+
collectGarbageBeforeRefusal,
|
|
174
|
+
collectYoungGenerationBeforeRefusal,
|
|
175
|
+
onFullChange: () => {
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
return {
|
|
179
|
+
fractions,
|
|
180
|
+
monitor,
|
|
181
|
+
refuseWhileFull(indexName) {
|
|
182
|
+
monitor.measureIfOlderThan(ENGINE_MEMORY_FIGURES_MAX_AGE_MS);
|
|
183
|
+
if (!monitor.isFull()) return;
|
|
184
|
+
monitor.measureWithoutGarbage();
|
|
185
|
+
if (!monitor.isFull()) return;
|
|
186
|
+
const report = monitor.report();
|
|
187
|
+
throw new NarsilError(
|
|
188
|
+
ErrorCodes.MEMORY_WRITE_BLOCKED,
|
|
189
|
+
`The engine refuses inserts and updates to index '${indexName}' because its memory passed the block level of its limit. Remove documents to free memory, and writes resume once memory falls below the resume level.`,
|
|
190
|
+
{
|
|
191
|
+
reason: MEMORY_FULL_REASON,
|
|
192
|
+
indexName,
|
|
193
|
+
processBytes: finiteOrNull(report?.processBytes),
|
|
194
|
+
processLimitBytes: finiteOrNull(report?.processLimitBytes),
|
|
195
|
+
heapBytes: finiteOrNull(report?.heapBytes),
|
|
196
|
+
heapLimitBytes: finiteOrNull(report?.heapLimitBytes)
|
|
197
|
+
}
|
|
198
|
+
);
|
|
199
|
+
},
|
|
200
|
+
memoryReleased() {
|
|
201
|
+
if (monitor.isFull()) monitor.remeasureAfterRelease();
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// src/engine/memory/validate.ts
|
|
207
|
+
function requireFraction(setting, value) {
|
|
208
|
+
if (value === void 0) return;
|
|
209
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1) return;
|
|
210
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${setting} must be a number above 0 and at most 1`, {
|
|
211
|
+
[setting.slice(setting.lastIndexOf(".") + 1)]: value
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
function validateMemoryFractions(section, memory, extraFractions = []) {
|
|
215
|
+
if (memory === void 0) return;
|
|
216
|
+
if (typeof memory !== "object" || memory === null || Array.isArray(memory)) {
|
|
217
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${section} must be an object`, { memory });
|
|
218
|
+
}
|
|
219
|
+
const settings = memory;
|
|
220
|
+
for (const name of ["blockWritesAbove", "resumeWritesBelow", ...extraFractions]) {
|
|
221
|
+
requireFraction(`${section}.${name}`, settings[name]);
|
|
222
|
+
}
|
|
223
|
+
const block = typeof settings.blockWritesAbove === "number" ? settings.blockWritesAbove : DEFAULT_BLOCK_WRITES_ABOVE;
|
|
224
|
+
const resume = typeof settings.resumeWritesBelow === "number" ? settings.resumeWritesBelow : DEFAULT_RESUME_WRITES_BELOW;
|
|
225
|
+
if (resume >= block) {
|
|
226
|
+
throw new NarsilError(
|
|
227
|
+
ErrorCodes.CONFIG_INVALID,
|
|
228
|
+
`${section}.resumeWritesBelow (${resume}) must lie below blockWritesAbove (${block})`,
|
|
229
|
+
{ blockWritesAbove: block, resumeWritesBelow: resume }
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// src/engine/memory/write-block.ts
|
|
235
|
+
var MEMORY_CONFIG_KEYS = {
|
|
236
|
+
blockWritesAbove: true,
|
|
237
|
+
resumeWritesBelow: true
|
|
238
|
+
};
|
|
239
|
+
function validateMemorySettings(settings) {
|
|
240
|
+
if (settings !== null && typeof settings === "object") {
|
|
241
|
+
for (const key of Object.keys(settings)) {
|
|
242
|
+
if (Object.hasOwn(MEMORY_CONFIG_KEYS, key)) continue;
|
|
243
|
+
throw new NarsilError(
|
|
244
|
+
ErrorCodes.CONFIG_INVALID,
|
|
245
|
+
`The configuration sets "memory.${key}", which is no setting that createNarsil accepts. The settings at that level are ${Object.keys(MEMORY_CONFIG_KEYS).join(", ")}`,
|
|
246
|
+
{ setting: `memory.${key}`, settings: Object.keys(MEMORY_CONFIG_KEYS) }
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
validateMemoryFractions("memory", settings);
|
|
251
|
+
}
|
|
252
|
+
function createMemoryWriteBlock(deps) {
|
|
253
|
+
if (deps.capacityScope !== "index") return void 0;
|
|
254
|
+
const measured = createMeasuredWriteBlock(deps.settings);
|
|
255
|
+
return { refuseWhileFull: measured.refuseWhileFull, memoryReleased: measured.memoryReleased };
|
|
256
|
+
}
|
|
257
|
+
|
|
22
258
|
// src/engine/config-keys.ts
|
|
23
259
|
var NARSIL_CONFIG_KEYS = {
|
|
24
260
|
persistence: true,
|
|
@@ -35,7 +271,8 @@ var NARSIL_CONFIG_KEYS = {
|
|
|
35
271
|
analysis: true,
|
|
36
272
|
lifecycle: true,
|
|
37
273
|
patternWorkCap: true,
|
|
38
|
-
events: true
|
|
274
|
+
events: true,
|
|
275
|
+
memory: true
|
|
39
276
|
};
|
|
40
277
|
var EVENT_NAMES = {
|
|
41
278
|
write: true,
|
|
@@ -113,6 +350,7 @@ function requireKnownConfig(config) {
|
|
|
113
350
|
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
114
351
|
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
115
352
|
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
353
|
+
validateMemorySettings(config.memory);
|
|
116
354
|
requireNamedPlugins(config.plugins);
|
|
117
355
|
requireValidPatternWorkCap(config.patternWorkCap);
|
|
118
356
|
requireEventListeners(config.events);
|
|
@@ -370,7 +608,7 @@ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
|
370
608
|
// src/partitioning/rewrite-spill-of.ts
|
|
371
609
|
async function rewriteSpillOf(storage, manager, target) {
|
|
372
610
|
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
373
|
-
const { openRewriteSpill } = await import('./module-
|
|
611
|
+
const { openRewriteSpill } = await import('./module-V5K4ZP23.mjs');
|
|
374
612
|
return openRewriteSpill(storage, manager, target);
|
|
375
613
|
}
|
|
376
614
|
|
|
@@ -914,7 +1152,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
914
1152
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
915
1153
|
if (existing !== void 0) {
|
|
916
1154
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
917
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
1155
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-ZCJNYBJM.mjs');
|
|
918
1156
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
919
1157
|
}
|
|
920
1158
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -2269,6 +2507,10 @@ function markCopiesLoaded(state, indexName, copyLoad) {
|
|
|
2269
2507
|
requestId: `copy-loaded-${copyLoad}`
|
|
2270
2508
|
});
|
|
2271
2509
|
}
|
|
2510
|
+
function copiesWillReceive(state, indexName) {
|
|
2511
|
+
if (state.copyLoadBuffers.has(indexName) || state.replicationQueues.has(indexName)) return true;
|
|
2512
|
+
return state.workerPool !== null && state.scaledOutIndexes.has(indexName);
|
|
2513
|
+
}
|
|
2272
2514
|
async function replicateToWorkers(state, action) {
|
|
2273
2515
|
if (!("indexName" in action)) {
|
|
2274
2516
|
if (state.workerPool === null) return;
|
|
@@ -2276,6 +2518,7 @@ async function replicateToWorkers(state, action) {
|
|
|
2276
2518
|
await dispatchToWorkers(state, action);
|
|
2277
2519
|
return;
|
|
2278
2520
|
}
|
|
2521
|
+
if (!copiesWillReceive(state, action.indexName)) return;
|
|
2279
2522
|
const detached = detachCallerDocuments(state, action);
|
|
2280
2523
|
const loading = state.copyLoadBuffers.get(action.indexName);
|
|
2281
2524
|
if (loading !== void 0) {
|
|
@@ -2765,297 +3008,6 @@ async function createWorkerFactory(entryPoint) {
|
|
|
2765
3008
|
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
|
|
2766
3009
|
}
|
|
2767
3010
|
|
|
2768
|
-
// src/workers/pool-memory.ts
|
|
2769
|
-
function toFinite(value) {
|
|
2770
|
-
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
2771
|
-
}
|
|
2772
|
-
function toHeapLimit(value) {
|
|
2773
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
2774
|
-
}
|
|
2775
|
-
async function memoryStatsOf(slots) {
|
|
2776
|
-
const indices = [];
|
|
2777
|
-
const pending = [];
|
|
2778
|
-
for (const [slotIndex, slot] of slots) {
|
|
2779
|
-
indices.push(slotIndex);
|
|
2780
|
-
pending.push(
|
|
2781
|
-
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
2782
|
-
);
|
|
2783
|
-
}
|
|
2784
|
-
const reports = await Promise.all(pending);
|
|
2785
|
-
const stats = [];
|
|
2786
|
-
for (let n = 0; n < indices.length; n++) {
|
|
2787
|
-
const report = reports[n];
|
|
2788
|
-
stats.push({
|
|
2789
|
-
workerId: indices[n],
|
|
2790
|
-
heapUsed: toFinite(report?.heapUsed),
|
|
2791
|
-
heapTotal: toFinite(report?.heapTotal),
|
|
2792
|
-
heapLimit: toHeapLimit(report?.heapLimit),
|
|
2793
|
-
external: toFinite(report?.external)
|
|
2794
|
-
});
|
|
2795
|
-
}
|
|
2796
|
-
return stats;
|
|
2797
|
-
}
|
|
2798
|
-
|
|
2799
|
-
// src/workers/pool.ts
|
|
2800
|
-
function createWorkerPool(config) {
|
|
2801
|
-
const workerCount = resolveWorkerCount(config.count);
|
|
2802
|
-
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
2803
|
-
const workers = /* @__PURE__ */ new Map();
|
|
2804
|
-
const deadSlots = /* @__PURE__ */ new Set();
|
|
2805
|
-
const goneSlots = /* @__PURE__ */ new Set();
|
|
2806
|
-
const threadsStillRunning = /* @__PURE__ */ new Set();
|
|
2807
|
-
const waitingForEveryThread = [];
|
|
2808
|
-
const indexAssignment = /* @__PURE__ */ new Map();
|
|
2809
|
-
let isShutdown = false;
|
|
2810
|
-
function handleWorkerDeath(slotIndex, error) {
|
|
2811
|
-
if (isShutdown) {
|
|
2812
|
-
return;
|
|
2813
|
-
}
|
|
2814
|
-
const slot = workers.get(slotIndex);
|
|
2815
|
-
if (slot !== void 0 && !slot.serving) {
|
|
2816
|
-
workers.delete(slotIndex);
|
|
2817
|
-
goneSlots.delete(slotIndex);
|
|
2818
|
-
return;
|
|
2819
|
-
}
|
|
2820
|
-
if (deadSlots.has(slotIndex)) {
|
|
2821
|
-
return;
|
|
2822
|
-
}
|
|
2823
|
-
deadSlots.add(slotIndex);
|
|
2824
|
-
const indexNames = slot ? [...slot.indexes].sort() : [];
|
|
2825
|
-
workers.delete(slotIndex);
|
|
2826
|
-
config.onWorkerCrash?.(slotIndex, indexNames, error);
|
|
2827
|
-
}
|
|
2828
|
-
function handleWorkerGone(slotIndex) {
|
|
2829
|
-
if (!deadSlots.has(slotIndex) || workers.has(slotIndex)) {
|
|
2830
|
-
return;
|
|
2831
|
-
}
|
|
2832
|
-
goneSlots.add(slotIndex);
|
|
2833
|
-
config.onWorkerGone?.(slotIndex);
|
|
2834
|
-
}
|
|
2835
|
-
function spawnSlot(slotIndex, serving) {
|
|
2836
|
-
const slot = {
|
|
2837
|
-
executor: config.workerFactory(
|
|
2838
|
-
slotIndex,
|
|
2839
|
-
(error) => handleWorkerDeath(slotIndex, error),
|
|
2840
|
-
() => {
|
|
2841
|
-
noteThreadGone(slot.executor);
|
|
2842
|
-
handleWorkerGone(slotIndex);
|
|
2843
|
-
}
|
|
2844
|
-
),
|
|
2845
|
-
indexes: /* @__PURE__ */ new Set(),
|
|
2846
|
-
inFlight: 0,
|
|
2847
|
-
serving
|
|
2848
|
-
};
|
|
2849
|
-
threadsStillRunning.add(slot.executor);
|
|
2850
|
-
workers.set(slotIndex, slot);
|
|
2851
|
-
return slot;
|
|
2852
|
-
}
|
|
2853
|
-
function noteThreadGone(executor) {
|
|
2854
|
-
threadsStillRunning.delete(executor);
|
|
2855
|
-
if (threadsStillRunning.size > 0) return;
|
|
2856
|
-
for (const resume of waitingForEveryThread.splice(0)) resume();
|
|
2857
|
-
}
|
|
2858
|
-
function whenEveryThreadIsGone() {
|
|
2859
|
-
if (threadsStillRunning.size === 0) return Promise.resolve();
|
|
2860
|
-
return new Promise((resolve) => {
|
|
2861
|
-
waitingForEveryThread.push(resolve);
|
|
2862
|
-
});
|
|
2863
|
-
}
|
|
2864
|
-
function shutDownUntilTheThreadIsGone(executor) {
|
|
2865
|
-
return executor.shutdown().then(() => noteThreadGone(executor));
|
|
2866
|
-
}
|
|
2867
|
-
function ensureWorker(slotIndex) {
|
|
2868
|
-
const slot = workers.get(slotIndex);
|
|
2869
|
-
if (slot !== void 0) return slot;
|
|
2870
|
-
if (deadSlots.has(slotIndex)) return void 0;
|
|
2871
|
-
return spawnSlot(slotIndex, true);
|
|
2872
|
-
}
|
|
2873
|
-
function deadWorkerIds() {
|
|
2874
|
-
return [...deadSlots].filter((slotIndex) => goneSlots.has(slotIndex) && !workers.has(slotIndex)).sort((a, b) => a - b);
|
|
2875
|
-
}
|
|
2876
|
-
function spawnReplacement(slotIndex) {
|
|
2877
|
-
if (isShutdown || !deadSlots.has(slotIndex) || !goneSlots.has(slotIndex) || workers.has(slotIndex)) return null;
|
|
2878
|
-
const slot = spawnSlot(slotIndex, false);
|
|
2879
|
-
return {
|
|
2880
|
-
workerId: slotIndex,
|
|
2881
|
-
executor: slot.executor,
|
|
2882
|
-
hold(indexName) {
|
|
2883
|
-
slot.indexes.add(indexName);
|
|
2884
|
-
},
|
|
2885
|
-
admit() {
|
|
2886
|
-
if (workers.get(slotIndex) !== slot) return;
|
|
2887
|
-
deadSlots.delete(slotIndex);
|
|
2888
|
-
goneSlots.delete(slotIndex);
|
|
2889
|
-
slot.serving = true;
|
|
2890
|
-
},
|
|
2891
|
-
abandon() {
|
|
2892
|
-
if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
|
|
2893
|
-
void shutDownUntilTheThreadIsGone(slot.executor).catch(() => void 0);
|
|
2894
|
-
}
|
|
2895
|
-
};
|
|
2896
|
-
}
|
|
2897
|
-
function isLongQueryWorker(workerId) {
|
|
2898
|
-
return workerId >= queryWorkerCount;
|
|
2899
|
-
}
|
|
2900
|
-
function assignSlot(indexName) {
|
|
2901
|
-
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
2902
|
-
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
2903
|
-
const candidate2 = (preferred + probe) % queryWorkerCount;
|
|
2904
|
-
if (!deadSlots.has(candidate2)) {
|
|
2905
|
-
return candidate2;
|
|
2906
|
-
}
|
|
2907
|
-
}
|
|
2908
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
|
|
2909
|
-
}
|
|
2910
|
-
function addIndex(indexName) {
|
|
2911
|
-
if (isShutdown) {
|
|
2912
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
2913
|
-
}
|
|
2914
|
-
if (indexAssignment.has(indexName)) {
|
|
2915
|
-
return;
|
|
2916
|
-
}
|
|
2917
|
-
const slotIndex = assignSlot(indexName);
|
|
2918
|
-
const slot = ensureWorker(slotIndex);
|
|
2919
|
-
if (!slot) {
|
|
2920
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
|
|
2921
|
-
}
|
|
2922
|
-
slot.indexes.add(indexName);
|
|
2923
|
-
indexAssignment.set(indexName, slotIndex);
|
|
2924
|
-
}
|
|
2925
|
-
function removeIndex(indexName) {
|
|
2926
|
-
for (const slot of workers.values()) slot.indexes.delete(indexName);
|
|
2927
|
-
indexAssignment.delete(indexName);
|
|
2928
|
-
}
|
|
2929
|
-
function getExecutor(indexName) {
|
|
2930
|
-
if (isShutdown) {
|
|
2931
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
2932
|
-
}
|
|
2933
|
-
const slotIndex = indexAssignment.get(indexName);
|
|
2934
|
-
if (slotIndex === void 0) {
|
|
2935
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
2936
|
-
}
|
|
2937
|
-
const slot = workers.get(slotIndex);
|
|
2938
|
-
if (slot?.serving) {
|
|
2939
|
-
return slot.executor;
|
|
2940
|
-
}
|
|
2941
|
-
for (const survivor of workers.values()) {
|
|
2942
|
-
if (survivor.serving && survivor.indexes.has(indexName)) {
|
|
2943
|
-
return survivor.executor;
|
|
2944
|
-
}
|
|
2945
|
-
}
|
|
2946
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
2947
|
-
}
|
|
2948
|
-
async function shutdown() {
|
|
2949
|
-
if (isShutdown) {
|
|
2950
|
-
return;
|
|
2951
|
-
}
|
|
2952
|
-
isShutdown = true;
|
|
2953
|
-
const shutdownPromises = [...threadsStillRunning].map((executor) => {
|
|
2954
|
-
let timeoutId;
|
|
2955
|
-
const timeoutPromise = new Promise((resolve) => {
|
|
2956
|
-
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
2957
|
-
});
|
|
2958
|
-
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
2959
|
-
() => clearTimeout(timeoutId)
|
|
2960
|
-
);
|
|
2961
|
-
});
|
|
2962
|
-
await Promise.allSettled(shutdownPromises);
|
|
2963
|
-
indexAssignment.clear();
|
|
2964
|
-
}
|
|
2965
|
-
function getAllExecutors() {
|
|
2966
|
-
return [...workers.values()].map((slot) => slot.executor);
|
|
2967
|
-
}
|
|
2968
|
-
function executorEntries() {
|
|
2969
|
-
return [...workers].map(([workerId, slot]) => ({ workerId, executor: slot.executor }));
|
|
2970
|
-
}
|
|
2971
|
-
function executorsHolding(indexName) {
|
|
2972
|
-
const holders = [];
|
|
2973
|
-
for (const slot of workers.values()) {
|
|
2974
|
-
if (slot.indexes.has(indexName)) holders.push(slot.executor);
|
|
2975
|
-
}
|
|
2976
|
-
return holders;
|
|
2977
|
-
}
|
|
2978
|
-
function lease(workerId, slot) {
|
|
2979
|
-
slot.inFlight += 1;
|
|
2980
|
-
let released = false;
|
|
2981
|
-
return {
|
|
2982
|
-
workerId,
|
|
2983
|
-
executor: slot.executor,
|
|
2984
|
-
release() {
|
|
2985
|
-
if (released) return;
|
|
2986
|
-
released = true;
|
|
2987
|
-
slot.inFlight -= 1;
|
|
2988
|
-
}
|
|
2989
|
-
};
|
|
2990
|
-
}
|
|
2991
|
-
function leaseLeastBusy(indexName) {
|
|
2992
|
-
let chosenId = -1;
|
|
2993
|
-
let chosen = null;
|
|
2994
|
-
for (const [workerId, slot] of workers) {
|
|
2995
|
-
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
2996
|
-
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
2997
|
-
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
2998
|
-
chosenId = workerId;
|
|
2999
|
-
chosen = slot;
|
|
3000
|
-
}
|
|
3001
|
-
if (chosen.inFlight === 0) break;
|
|
3002
|
-
}
|
|
3003
|
-
return chosen === null ? null : lease(chosenId, chosen);
|
|
3004
|
-
}
|
|
3005
|
-
function queriesInFlight() {
|
|
3006
|
-
let total = 0;
|
|
3007
|
-
for (const [workerId, slot] of workers) {
|
|
3008
|
-
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
3009
|
-
}
|
|
3010
|
-
return total;
|
|
3011
|
-
}
|
|
3012
|
-
function leaseIdle(limit) {
|
|
3013
|
-
const leases = [];
|
|
3014
|
-
for (const [workerId, slot] of workers) {
|
|
3015
|
-
if (leases.length >= limit) break;
|
|
3016
|
-
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
3017
|
-
}
|
|
3018
|
-
return leases;
|
|
3019
|
-
}
|
|
3020
|
-
function addIndexToAll(indexName) {
|
|
3021
|
-
if (isShutdown) {
|
|
3022
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
3023
|
-
}
|
|
3024
|
-
for (let i = 0; i < workerCount; i++) {
|
|
3025
|
-
ensureWorker(i)?.indexes.add(indexName);
|
|
3026
|
-
}
|
|
3027
|
-
indexAssignment.set(indexName, assignSlot(indexName));
|
|
3028
|
-
}
|
|
3029
|
-
function spawnAll() {
|
|
3030
|
-
if (isShutdown) {
|
|
3031
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
3032
|
-
}
|
|
3033
|
-
for (let i = 0; i < workerCount; i++) ensureWorker(i);
|
|
3034
|
-
}
|
|
3035
|
-
return {
|
|
3036
|
-
getExecutor,
|
|
3037
|
-
getAllExecutors,
|
|
3038
|
-
executorEntries,
|
|
3039
|
-
executorsHolding,
|
|
3040
|
-
leaseLeastBusy,
|
|
3041
|
-
leaseIdle,
|
|
3042
|
-
isLongQueryWorker,
|
|
3043
|
-
queriesInFlight,
|
|
3044
|
-
spawnAll,
|
|
3045
|
-
deadWorkerIds,
|
|
3046
|
-
spawnReplacement,
|
|
3047
|
-
get workerCount() {
|
|
3048
|
-
return workerCount;
|
|
3049
|
-
},
|
|
3050
|
-
addIndex,
|
|
3051
|
-
addIndexToAll,
|
|
3052
|
-
removeIndex,
|
|
3053
|
-
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
3054
|
-
shutdown,
|
|
3055
|
-
whenEveryThreadIsGone
|
|
3056
|
-
};
|
|
3057
|
-
}
|
|
3058
|
-
|
|
3059
3011
|
// src/engine/orchestration/vector-copies.ts
|
|
3060
3012
|
function holdsIndex(state, indexName) {
|
|
3061
3013
|
return state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName);
|
|
@@ -3345,7 +3297,7 @@ async function startPool(state) {
|
|
|
3345
3297
|
);
|
|
3346
3298
|
}
|
|
3347
3299
|
eligibleIndexNames(state);
|
|
3348
|
-
const factory = await createWorkerFactory();
|
|
3300
|
+
const [factory, { createWorkerPool }] = await Promise.all([createWorkerFactory(), import('./pool-UPV5PVPY.mjs')]);
|
|
3349
3301
|
let started = null;
|
|
3350
3302
|
const pool = createWorkerPool({
|
|
3351
3303
|
count: state.keywordWorkerCount + state.longQueryWorkerCount,
|
|
@@ -4036,7 +3988,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
4036
3988
|
// src/engine/core.ts
|
|
4037
3989
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
4038
3990
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
4039
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
3991
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-L2DZYY6N.mjs')).resolveRerankSettings(config);
|
|
4040
3992
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
4041
3993
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
4042
3994
|
const vectorCopyPolicy = {
|
|
@@ -4047,7 +3999,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4047
3999
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4048
4000
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4049
4001
|
}
|
|
4050
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4002
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-GNWQSZI3.mjs');
|
|
4051
4003
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4052
4004
|
const executor = createDirectExecutor({
|
|
4053
4005
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4232,7 +4184,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4232
4184
|
requireManager,
|
|
4233
4185
|
resumeChange: async (indexName) => {
|
|
4234
4186
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
4235
|
-
await (await import('./analysis-change-entry-
|
|
4187
|
+
await (await import('./analysis-change-entry-YN3ELL6C.mjs')).resumeChangeOf(engineCore, indexName);
|
|
4236
4188
|
},
|
|
4237
4189
|
onClose: hooks?.onIndexClose,
|
|
4238
4190
|
onOpen: (indexName) => {
|
|
@@ -4259,7 +4211,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4259
4211
|
bufferedDocState: (indexName, docId) => waqMap.get(indexName)?.bufferedDocState(docId),
|
|
4260
4212
|
capacityScope,
|
|
4261
4213
|
checkWatermark: watermarkNotifier.check,
|
|
4262
|
-
checkHeapPressure: heapPressureNotifier.check
|
|
4214
|
+
checkHeapPressure: heapPressureNotifier.check,
|
|
4215
|
+
memoryWriteBlock: createMemoryWriteBlock({
|
|
4216
|
+
settings: config?.memory,
|
|
4217
|
+
capacityScope})
|
|
4263
4218
|
};
|
|
4264
4219
|
const rebalanceCtx = {
|
|
4265
4220
|
router: rebalanceRouter,
|
|
@@ -4314,7 +4269,7 @@ async function startEngineCore(core, config) {
|
|
|
4314
4269
|
checkHeapAfterRecovery(core);
|
|
4315
4270
|
}
|
|
4316
4271
|
if (core.invalidation) await core.invalidation.start();
|
|
4317
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
4272
|
+
if (core.durability) await (await import('./analysis-change-entry-YN3ELL6C.mjs')).resumeEveryChange(core);
|
|
4318
4273
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
4319
4274
|
} catch (err) {
|
|
4320
4275
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -4352,7 +4307,7 @@ async function shutdownEngine(core) {
|
|
|
4352
4307
|
}
|
|
4353
4308
|
}
|
|
4354
4309
|
if (core.rerankAdapters.size > 0) {
|
|
4355
|
-
await (await import('./runtime-
|
|
4310
|
+
await (await import('./runtime-L2DZYY6N.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
4356
4311
|
}
|
|
4357
4312
|
await orchestrator.shutdown();
|
|
4358
4313
|
await executor.shutdown();
|
|
@@ -4591,34 +4546,17 @@ function loadFailed(message, details) {
|
|
|
4591
4546
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
|
|
4592
4547
|
}
|
|
4593
4548
|
async function gunzipWithinLimit(compressed, what) {
|
|
4594
|
-
const
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
let step;
|
|
4600
|
-
try {
|
|
4601
|
-
step = await reader.read();
|
|
4602
|
-
} catch (error) {
|
|
4603
|
-
loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
|
|
4604
|
-
cause: error instanceof Error ? error.message : String(error)
|
|
4605
|
-
});
|
|
4606
|
-
}
|
|
4607
|
-
if (step.done) break;
|
|
4608
|
-
total += step.value.byteLength;
|
|
4609
|
-
if (total > MAX_FILE_BYTES) {
|
|
4610
|
-
await reader.cancel();
|
|
4549
|
+
const { gunzip } = await import('./gzip-F4HMIEEQ.mjs');
|
|
4550
|
+
try {
|
|
4551
|
+
return await gunzip(compressed, MAX_FILE_BYTES);
|
|
4552
|
+
} catch (error) {
|
|
4553
|
+
if (error?.code === "EFBIG") {
|
|
4611
4554
|
loadFailed(`The ${what} inflates past the file size limit of ${MAX_FILE_BYTES} bytes`);
|
|
4612
4555
|
}
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
let offset = 0;
|
|
4617
|
-
for (const chunk of chunks) {
|
|
4618
|
-
inflated.set(chunk, offset);
|
|
4619
|
-
offset += chunk.byteLength;
|
|
4556
|
+
loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
|
|
4557
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
4558
|
+
});
|
|
4620
4559
|
}
|
|
4621
|
-
return inflated;
|
|
4622
4560
|
}
|
|
4623
4561
|
async function openChecksummedEnvelope(data, expected) {
|
|
4624
4562
|
const { envelopeFormatVersion, what } = expected;
|
|
@@ -4918,7 +4856,7 @@ async function dropEngineIndex(core, name) {
|
|
|
4918
4856
|
const entry = core.requireIndex(name);
|
|
4919
4857
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
4920
4858
|
const abandonChange = async () => {
|
|
4921
|
-
await (await import('./analysis-change-entry-
|
|
4859
|
+
await (await import('./analysis-change-entry-YN3ELL6C.mjs')).abandonChangeOf(core, name);
|
|
4922
4860
|
};
|
|
4923
4861
|
if (changeRecorded()) await abandonChange();
|
|
4924
4862
|
await core.indexState.drop(name, async () => {
|
|
@@ -5204,18 +5142,24 @@ async function rollbackInsertedDocument(ctx, indexName, docId, inserted, origina
|
|
|
5204
5142
|
return;
|
|
5205
5143
|
}
|
|
5206
5144
|
try {
|
|
5207
|
-
await
|
|
5208
|
-
|
|
5145
|
+
await changeIndexOutsideExternalWaits(async () => {
|
|
5146
|
+
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
|
|
5147
|
+
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
5148
|
+
});
|
|
5209
5149
|
} catch (rollbackError) {
|
|
5210
5150
|
rollbackFailed(docId, originalError, rollbackError);
|
|
5211
5151
|
}
|
|
5212
5152
|
}
|
|
5213
5153
|
async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, originalError) {
|
|
5214
|
-
if (restoreDoc === void 0
|
|
5154
|
+
if (restoreDoc === void 0) {
|
|
5215
5155
|
return;
|
|
5216
5156
|
}
|
|
5157
|
+
const manager = ctx.requireManager(indexName);
|
|
5217
5158
|
try {
|
|
5218
|
-
await
|
|
5159
|
+
await changeIndexOutsideExternalWaits(async () => {
|
|
5160
|
+
if (manager.has(docId)) return;
|
|
5161
|
+
await ctx.executor.execute({ type: "insert", indexName, docId, document: restoreDoc, requestId: docId });
|
|
5162
|
+
});
|
|
5219
5163
|
} catch (rollbackError) {
|
|
5220
5164
|
rollbackFailed(docId, originalError, rollbackError);
|
|
5221
5165
|
}
|
|
@@ -5225,7 +5169,9 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
5225
5169
|
return;
|
|
5226
5170
|
}
|
|
5227
5171
|
try {
|
|
5228
|
-
await
|
|
5172
|
+
await changeIndexOutsideExternalWaits(async () => {
|
|
5173
|
+
await ctx.executor.execute({ type: "update", indexName, docId, document: restoreDoc, requestId: docId });
|
|
5174
|
+
});
|
|
5229
5175
|
} catch (rollbackError) {
|
|
5230
5176
|
rollbackFailed(docId, originalError, rollbackError);
|
|
5231
5177
|
}
|
|
@@ -5406,7 +5352,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
5406
5352
|
}
|
|
5407
5353
|
let inserted = false;
|
|
5408
5354
|
let buffered = false;
|
|
5409
|
-
const
|
|
5355
|
+
const insertIntoIndex = async () => {
|
|
5410
5356
|
admitInsert(ctx, indexName, insertManager, resolvedDocId);
|
|
5411
5357
|
if (ctx.bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
|
|
5412
5358
|
buffered = true;
|
|
@@ -5441,6 +5387,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
5441
5387
|
throw err;
|
|
5442
5388
|
}
|
|
5443
5389
|
};
|
|
5390
|
+
const applyInsert = indexChangeOutsideExternalWaits(insertIntoIndex);
|
|
5444
5391
|
if (ctx.durability) {
|
|
5445
5392
|
try {
|
|
5446
5393
|
await ctx.durability.recordInsertOrUpdate(indexName, resolvedDocId, document, applyInsert);
|
|
@@ -5495,6 +5442,7 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
5495
5442
|
}
|
|
5496
5443
|
const failures = [];
|
|
5497
5444
|
for (const apply of applies) {
|
|
5445
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5498
5446
|
try {
|
|
5499
5447
|
await apply();
|
|
5500
5448
|
failures.push(null);
|
|
@@ -5508,7 +5456,7 @@ async function recordChunk(ctx, indexName, mutations, applies) {
|
|
|
5508
5456
|
// src/engine/mutations/insert-batch-apply.ts
|
|
5509
5457
|
function applyOf(ctx, indexName, doc, options, progress) {
|
|
5510
5458
|
const manager = ctx.requireManager(indexName);
|
|
5511
|
-
|
|
5459
|
+
const insertIntoIndex = async () => {
|
|
5512
5460
|
if (ctx.abortController.signal.aborted) {
|
|
5513
5461
|
progress.skipped = true;
|
|
5514
5462
|
throw ctx.abortController.signal.reason;
|
|
@@ -5547,6 +5495,7 @@ function applyOf(ctx, indexName, doc, options, progress) {
|
|
|
5547
5495
|
throw vecErr;
|
|
5548
5496
|
}
|
|
5549
5497
|
};
|
|
5498
|
+
return indexChangeOutsideExternalWaits(insertIntoIndex);
|
|
5550
5499
|
}
|
|
5551
5500
|
async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
5552
5501
|
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
@@ -5832,7 +5781,7 @@ function attachOnce(manager, attachment) {
|
|
|
5832
5781
|
}
|
|
5833
5782
|
function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
5834
5783
|
const manager = ctx.requireManager(indexName);
|
|
5835
|
-
|
|
5784
|
+
const attachToIndex = async () => {
|
|
5836
5785
|
attachOnce(manager, attachment);
|
|
5837
5786
|
progress.applied = true;
|
|
5838
5787
|
try {
|
|
@@ -5855,6 +5804,7 @@ function applyOfMergedDocument(ctx, indexName, doc, attachment, progress) {
|
|
|
5855
5804
|
throw vecErr;
|
|
5856
5805
|
}
|
|
5857
5806
|
};
|
|
5807
|
+
return indexChangeOutsideExternalWaits(attachToIndex);
|
|
5858
5808
|
}
|
|
5859
5809
|
function chunksOfWholeSegments(built, frozen, memberIndexes, admitted) {
|
|
5860
5810
|
const chunks = [];
|
|
@@ -6040,6 +5990,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
|
6040
5990
|
async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
6041
5991
|
ctx.guardShutdown();
|
|
6042
5992
|
const entry = ctx.requireIndex(indexName);
|
|
5993
|
+
validateBatchList(documents, "documents");
|
|
6043
5994
|
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
|
|
6044
5995
|
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
6045
5996
|
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
@@ -6104,6 +6055,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
6104
6055
|
}
|
|
6105
6056
|
const prepared = [];
|
|
6106
6057
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
6058
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
6107
6059
|
if (ctx.abortController.signal.aborted) break;
|
|
6108
6060
|
if (chunkFailedIndexes.has(i)) continue;
|
|
6109
6061
|
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
@@ -6147,6 +6099,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
6147
6099
|
const replicatedAsSegments = copiesBuildSegments && await replicateAsSegments(ctx, indexName, replicableIds, replicableDocs, options?.skipClone);
|
|
6148
6100
|
if (!replicatedAsSegments) {
|
|
6149
6101
|
for (let i = 0; i < replicableIds.length; i++) {
|
|
6102
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
6150
6103
|
await ctx.orchestrator.replicateToWorkers({
|
|
6151
6104
|
type: "insert",
|
|
6152
6105
|
indexName,
|
|
@@ -6194,7 +6147,7 @@ async function prepareRemoval(ctx, indexName, docId) {
|
|
|
6194
6147
|
}
|
|
6195
6148
|
function applyOfRemoval(ctx, indexName, prepared) {
|
|
6196
6149
|
const { docId, removedSpellings } = prepared;
|
|
6197
|
-
|
|
6150
|
+
const removeFromIndex = async () => {
|
|
6198
6151
|
if (ctx.isRebalancing(indexName)) {
|
|
6199
6152
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
6200
6153
|
const exists = bufferedState !== void 0 ? bufferedState === "present" : ctx.requireManager(indexName).has(docId);
|
|
@@ -6209,6 +6162,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
6209
6162
|
await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId, removedSpellings });
|
|
6210
6163
|
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
6211
6164
|
};
|
|
6165
|
+
return indexChangeOutsideExternalWaits(removeFromIndex);
|
|
6212
6166
|
}
|
|
6213
6167
|
async function settleRemoval(ctx, indexName, prepared) {
|
|
6214
6168
|
const { docId, removedSpellings } = prepared;
|
|
@@ -6246,6 +6200,7 @@ async function removeDocument(ctx, indexName, docId, options) {
|
|
|
6246
6200
|
async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
6247
6201
|
ctx.guardShutdown();
|
|
6248
6202
|
ctx.requireIndex(indexName);
|
|
6203
|
+
validateBatchList(docIds, "document IDs");
|
|
6249
6204
|
const succeeded = [];
|
|
6250
6205
|
const failed = [];
|
|
6251
6206
|
const manager = ctx.requireManager(indexName);
|
|
@@ -6255,6 +6210,7 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
6255
6210
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
6256
6211
|
const prepared = [];
|
|
6257
6212
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
6213
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
6258
6214
|
try {
|
|
6259
6215
|
prepared.push(await prepareRemoval(ctx, indexName, docIds[i]));
|
|
6260
6216
|
} catch (err) {
|
|
@@ -6399,7 +6355,7 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
|
6399
6355
|
function applyOfUpdate(ctx, indexName, prepared) {
|
|
6400
6356
|
const manager = ctx.requireManager(indexName);
|
|
6401
6357
|
const { docId, document, partitionDoc, rollbackDoc, extractedVectors, removedSpellings } = prepared;
|
|
6402
|
-
|
|
6358
|
+
const updateInIndex = async () => {
|
|
6403
6359
|
if (ctx.isRebalancing(indexName)) {
|
|
6404
6360
|
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
6405
6361
|
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
@@ -6435,6 +6391,7 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
6435
6391
|
throw err;
|
|
6436
6392
|
}
|
|
6437
6393
|
};
|
|
6394
|
+
return indexChangeOutsideExternalWaits(updateInIndex);
|
|
6438
6395
|
}
|
|
6439
6396
|
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
6440
6397
|
ctx.guardShutdown();
|
|
@@ -6474,6 +6431,7 @@ function requireUpdateEntry(update) {
|
|
|
6474
6431
|
async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
6475
6432
|
ctx.guardShutdown();
|
|
6476
6433
|
const entry = ctx.requireIndex(indexName);
|
|
6434
|
+
validateBatchList(updates, "updates");
|
|
6477
6435
|
const succeeded = [];
|
|
6478
6436
|
const failed = [];
|
|
6479
6437
|
const updateBatchManager = ctx.requireManager(indexName);
|
|
@@ -6483,6 +6441,7 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
6483
6441
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
6484
6442
|
const prepared = [];
|
|
6485
6443
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
6444
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
6486
6445
|
const update = updates[i];
|
|
6487
6446
|
const docId = reportedDocIdOf(update);
|
|
6488
6447
|
try {
|
|
@@ -6526,51 +6485,130 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
6526
6485
|
return { succeeded, failed };
|
|
6527
6486
|
}
|
|
6528
6487
|
|
|
6529
|
-
// src/engine/
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
if (
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6488
|
+
// src/engine/memory/guarded-writes.ts
|
|
6489
|
+
var EMPTY_DOCUMENT = {};
|
|
6490
|
+
function blockFor(ctx, indexName) {
|
|
6491
|
+
const block = ctx.memoryWriteBlock;
|
|
6492
|
+
if (block === void 0) return void 0;
|
|
6493
|
+
if (block.grouping !== void 0) {
|
|
6494
|
+
ctx.guardShutdown();
|
|
6495
|
+
ctx.requireIndex(indexName);
|
|
6496
|
+
block.refuseWhileFull(indexName);
|
|
6497
|
+
return block;
|
|
6498
|
+
}
|
|
6499
|
+
try {
|
|
6500
|
+
block.refuseWhileFull(indexName);
|
|
6501
|
+
} catch (refusal) {
|
|
6502
|
+
ctx.guardShutdown();
|
|
6503
|
+
ctx.requireIndex(indexName);
|
|
6504
|
+
throw refusal;
|
|
6539
6505
|
}
|
|
6540
|
-
|
|
6541
|
-
single.set(fieldName, idx);
|
|
6542
|
-
return single;
|
|
6506
|
+
return block;
|
|
6543
6507
|
}
|
|
6544
|
-
function
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6508
|
+
async function withReservation(grouping, indexName, document, write) {
|
|
6509
|
+
if (grouping === void 0) return write();
|
|
6510
|
+
const reservation = grouping.reserveFor(indexName, document);
|
|
6511
|
+
try {
|
|
6512
|
+
return await write();
|
|
6513
|
+
} finally {
|
|
6514
|
+
reservation.release();
|
|
6548
6515
|
}
|
|
6549
6516
|
}
|
|
6550
|
-
async function
|
|
6551
|
-
const
|
|
6552
|
-
|
|
6553
|
-
|
|
6517
|
+
async function writeInGroups(grouping, indexName, items, write) {
|
|
6518
|
+
const succeeded = [];
|
|
6519
|
+
const failed = [];
|
|
6520
|
+
await grouping.applyInGroups(indexName, {
|
|
6521
|
+
items,
|
|
6522
|
+
failed,
|
|
6523
|
+
apply: async (group) => {
|
|
6524
|
+
const result = await write(group);
|
|
6525
|
+
for (const docId of result.succeeded) succeeded.push(docId);
|
|
6526
|
+
for (const failure of result.failed) failed.push(failure);
|
|
6527
|
+
}
|
|
6528
|
+
});
|
|
6529
|
+
return { succeeded, failed };
|
|
6530
|
+
}
|
|
6531
|
+
function updateItemOf(update) {
|
|
6532
|
+
const entry = update;
|
|
6533
|
+
if (entry === null || typeof entry !== "object") return { docId: "", document: EMPTY_DOCUMENT, update };
|
|
6534
|
+
const { docId, document } = entry;
|
|
6535
|
+
return {
|
|
6536
|
+
docId: typeof docId === "string" ? docId : "",
|
|
6537
|
+
document: document !== null && typeof document === "object" ? document : EMPTY_DOCUMENT,
|
|
6538
|
+
update
|
|
6539
|
+
};
|
|
6540
|
+
}
|
|
6541
|
+
async function insertDocument2(ctx, indexName, document, docId, options) {
|
|
6542
|
+
const block = blockFor(ctx, indexName);
|
|
6543
|
+
if (block === void 0) return insertDocument(ctx, indexName, document, docId, options);
|
|
6544
|
+
return withReservation(
|
|
6545
|
+
block.grouping,
|
|
6546
|
+
indexName,
|
|
6547
|
+
document,
|
|
6548
|
+
() => insertDocument(ctx, indexName, document, docId, options)
|
|
6549
|
+
);
|
|
6550
|
+
}
|
|
6551
|
+
async function insertDocumentBatch2(ctx, indexName, documents, options) {
|
|
6552
|
+
const block = blockFor(ctx, indexName);
|
|
6553
|
+
if (block === void 0) return insertDocumentBatch(ctx, indexName, documents, options);
|
|
6554
|
+
const grouping = block.grouping;
|
|
6555
|
+
if (grouping === void 0 || !Array.isArray(documents) || documents.length === 0) {
|
|
6556
|
+
return insertDocumentBatch(ctx, indexName, documents, options);
|
|
6554
6557
|
}
|
|
6558
|
+
const items = documents.map((document) => ({ docId: providedDocId(document) ?? "", document }));
|
|
6559
|
+
return writeInGroups(
|
|
6560
|
+
grouping,
|
|
6561
|
+
indexName,
|
|
6562
|
+
items,
|
|
6563
|
+
(group) => insertDocumentBatch(
|
|
6564
|
+
ctx,
|
|
6565
|
+
indexName,
|
|
6566
|
+
group.map((item) => item.document),
|
|
6567
|
+
options
|
|
6568
|
+
)
|
|
6569
|
+
);
|
|
6555
6570
|
}
|
|
6556
|
-
function
|
|
6557
|
-
const
|
|
6558
|
-
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
|
|
6562
|
-
|
|
6563
|
-
|
|
6564
|
-
|
|
6565
|
-
|
|
6566
|
-
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
6571
|
-
|
|
6571
|
+
async function updateDocument2(ctx, indexName, docId, document, options) {
|
|
6572
|
+
const block = blockFor(ctx, indexName);
|
|
6573
|
+
if (block === void 0) return updateDocument(ctx, indexName, docId, document, options);
|
|
6574
|
+
return withReservation(
|
|
6575
|
+
block.grouping,
|
|
6576
|
+
indexName,
|
|
6577
|
+
document,
|
|
6578
|
+
() => updateDocument(ctx, indexName, docId, document, options)
|
|
6579
|
+
);
|
|
6580
|
+
}
|
|
6581
|
+
async function updateDocumentBatch2(ctx, indexName, updates, options) {
|
|
6582
|
+
const block = blockFor(ctx, indexName);
|
|
6583
|
+
if (block === void 0) return updateDocumentBatch(ctx, indexName, updates, options);
|
|
6584
|
+
const grouping = block.grouping;
|
|
6585
|
+
if (grouping === void 0 || !Array.isArray(updates) || updates.length === 0) {
|
|
6586
|
+
return updateDocumentBatch(ctx, indexName, updates, options);
|
|
6572
6587
|
}
|
|
6573
|
-
return
|
|
6588
|
+
return writeInGroups(
|
|
6589
|
+
grouping,
|
|
6590
|
+
indexName,
|
|
6591
|
+
updates.map(updateItemOf),
|
|
6592
|
+
(group) => updateDocumentBatch(
|
|
6593
|
+
ctx,
|
|
6594
|
+
indexName,
|
|
6595
|
+
group.map((item) => item.update),
|
|
6596
|
+
options
|
|
6597
|
+
)
|
|
6598
|
+
);
|
|
6599
|
+
}
|
|
6600
|
+
async function removeDocument2(ctx, indexName, docId, options) {
|
|
6601
|
+
await removeDocument(ctx, indexName, docId, options);
|
|
6602
|
+
ctx.memoryWriteBlock?.memoryReleased();
|
|
6603
|
+
}
|
|
6604
|
+
async function removeDocumentBatch2(ctx, indexName, docIds, options) {
|
|
6605
|
+
const result = await removeDocumentBatch(ctx, indexName, docIds, options);
|
|
6606
|
+
if (result.succeeded.length > 0) ctx.memoryWriteBlock?.memoryReleased();
|
|
6607
|
+
return result;
|
|
6608
|
+
}
|
|
6609
|
+
async function dropEngineIndex2(core, name) {
|
|
6610
|
+
await dropEngineIndex(core, name);
|
|
6611
|
+
core.mutationCtx.memoryWriteBlock?.memoryReleased();
|
|
6574
6612
|
}
|
|
6575
6613
|
|
|
6576
6614
|
// src/runtime/process-memory.ts
|
|
@@ -6698,14 +6736,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6698
6736
|
return {
|
|
6699
6737
|
snapshot(indexName) {
|
|
6700
6738
|
return withOpenIndex2(indexName, async () => {
|
|
6701
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6739
|
+
const { createSnapshot } = await import('./snapshot-EHXTOTW5.mjs');
|
|
6702
6740
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6703
6741
|
});
|
|
6704
6742
|
},
|
|
6705
6743
|
async restore(indexName, data) {
|
|
6706
6744
|
core.guardShutdown();
|
|
6707
6745
|
validateIndexName(indexName);
|
|
6708
|
-
const restored = import('./restore-queue-
|
|
6746
|
+
const restored = import('./restore-queue-BQWWNNQI.mjs').then(
|
|
6709
6747
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6710
6748
|
);
|
|
6711
6749
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6714,8 +6752,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6714
6752
|
exportDocuments(indexName) {
|
|
6715
6753
|
return withOpenIndex2(indexName, async () => {
|
|
6716
6754
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6717
|
-
import('./snapshot-
|
|
6718
|
-
import('./write-
|
|
6755
|
+
import('./snapshot-EHXTOTW5.mjs'),
|
|
6756
|
+
import('./write-SDJXXQ2G.mjs')
|
|
6719
6757
|
]);
|
|
6720
6758
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6721
6759
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6723,7 +6761,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6723
6761
|
},
|
|
6724
6762
|
importDocumentExport(indexName, data) {
|
|
6725
6763
|
core.guardShutdown();
|
|
6726
|
-
const imported = import('./import-
|
|
6764
|
+
const imported = import('./import-DX55PN4G.mjs').then(
|
|
6727
6765
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
6728
6766
|
);
|
|
6729
6767
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -6834,7 +6872,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
6834
6872
|
const manager = core.requireManager(indexName);
|
|
6835
6873
|
const scoped = scopedParams(params, options);
|
|
6836
6874
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
6837
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
6875
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-L2DZYY6N.mjs');
|
|
6838
6876
|
const resolvedParams = await resolveVectorText(
|
|
6839
6877
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
6840
6878
|
entry.embeddingAdapter,
|
|
@@ -7030,6 +7068,9 @@ function createNarsilFromCore(core, config) {
|
|
|
7030
7068
|
const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
|
|
7031
7069
|
return emitWriteEventAfter(core, indexName, written);
|
|
7032
7070
|
}
|
|
7071
|
+
function withCountedWrite(indexName, action) {
|
|
7072
|
+
return countWrite(() => withOpenIndexWrite(indexName, action));
|
|
7073
|
+
}
|
|
7033
7074
|
const narsil = {
|
|
7034
7075
|
createIndex(name, indexConfig) {
|
|
7035
7076
|
return emitWriteEventAfter(
|
|
@@ -7045,7 +7086,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7045
7086
|
registerEngineRerankAdapter(core, name, adapter);
|
|
7046
7087
|
},
|
|
7047
7088
|
dropIndex(name) {
|
|
7048
|
-
return emitWriteEventAfter(core, name,
|
|
7089
|
+
return emitWriteEventAfter(core, name, dropEngineIndex2(core, name));
|
|
7049
7090
|
},
|
|
7050
7091
|
open(indexName) {
|
|
7051
7092
|
guardShutdown();
|
|
@@ -7091,22 +7132,22 @@ function createNarsilFromCore(core, config) {
|
|
|
7091
7132
|
return executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
7092
7133
|
},
|
|
7093
7134
|
insert(indexName, document, docId, options) {
|
|
7094
|
-
return
|
|
7135
|
+
return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
|
|
7095
7136
|
},
|
|
7096
7137
|
insertBatch(indexName, documents, options) {
|
|
7097
|
-
return
|
|
7138
|
+
return withCountedWrite(indexName, () => insertDocumentBatch2(mutationCtx, indexName, documents, options));
|
|
7098
7139
|
},
|
|
7099
7140
|
remove(indexName, docId, options) {
|
|
7100
|
-
return
|
|
7141
|
+
return withCountedWrite(indexName, () => removeDocument2(mutationCtx, indexName, docId, options));
|
|
7101
7142
|
},
|
|
7102
7143
|
removeBatch(indexName, docIds, options) {
|
|
7103
|
-
return
|
|
7144
|
+
return withCountedWrite(indexName, () => removeDocumentBatch2(mutationCtx, indexName, docIds, options));
|
|
7104
7145
|
},
|
|
7105
7146
|
update(indexName, docId, document, options) {
|
|
7106
|
-
return
|
|
7147
|
+
return withCountedWrite(indexName, () => updateDocument2(mutationCtx, indexName, docId, document, options));
|
|
7107
7148
|
},
|
|
7108
7149
|
updateBatch(indexName, updates, options) {
|
|
7109
|
-
return
|
|
7150
|
+
return withCountedWrite(indexName, () => updateDocumentBatch2(mutationCtx, indexName, updates, options));
|
|
7110
7151
|
},
|
|
7111
7152
|
waitForWrites(indexName) {
|
|
7112
7153
|
guardShutdown();
|
|
@@ -7156,7 +7197,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7156
7197
|
const entry = requireIndex(indexName);
|
|
7157
7198
|
await withOpenIndexWrite(indexName, async () => {
|
|
7158
7199
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
7159
|
-
return (await import('./analysis-change-
|
|
7200
|
+
return (await import('./analysis-change-ZCJNYBJM.mjs')).resumeAnalysisChange(core, indexName);
|
|
7160
7201
|
});
|
|
7161
7202
|
},
|
|
7162
7203
|
async changeAnalysis(indexName, change) {
|
|
@@ -7164,7 +7205,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7164
7205
|
requireIndex(indexName);
|
|
7165
7206
|
const changed = withOpenIndex2(
|
|
7166
7207
|
indexName,
|
|
7167
|
-
async () => (await import('./analysis-change-
|
|
7208
|
+
async () => (await import('./analysis-change-ZCJNYBJM.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
7168
7209
|
);
|
|
7169
7210
|
await emitWriteEventAfter(core, indexName, changed);
|
|
7170
7211
|
},
|
|
@@ -7202,7 +7243,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7202
7243
|
}
|
|
7203
7244
|
rebalancesStarting.set(indexName, started);
|
|
7204
7245
|
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
7205
|
-
const { executeRebalance } = await import('./analysis-change-
|
|
7246
|
+
const { executeRebalance } = await import('./analysis-change-ZCJNYBJM.mjs');
|
|
7206
7247
|
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
7207
7248
|
releaseWrites();
|
|
7208
7249
|
return running;
|
|
@@ -7271,6 +7312,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7271
7312
|
async compactVectors(indexName, fieldName) {
|
|
7272
7313
|
guardShutdown();
|
|
7273
7314
|
requireIndex(indexName);
|
|
7315
|
+
const { compactVectors } = await loadVectorCode();
|
|
7274
7316
|
await withOpenIndexWrite(indexName, async () => {
|
|
7275
7317
|
compactVectors(requireManager(indexName), indexName, fieldName);
|
|
7276
7318
|
});
|
|
@@ -7278,13 +7320,15 @@ function createNarsilFromCore(core, config) {
|
|
|
7278
7320
|
async optimizeVectors(indexName, fieldName) {
|
|
7279
7321
|
guardShutdown();
|
|
7280
7322
|
requireIndex(indexName);
|
|
7323
|
+
const { optimizeVectors } = await loadVectorCode();
|
|
7281
7324
|
await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
|
|
7282
7325
|
},
|
|
7283
7326
|
vectorMaintenanceStatus(indexName) {
|
|
7284
7327
|
guardShutdown();
|
|
7285
7328
|
requireIndex(indexName);
|
|
7286
7329
|
const manager = executor.getManager(indexName);
|
|
7287
|
-
|
|
7330
|
+
const vectorCode = loadedVectorCode();
|
|
7331
|
+
return manager === void 0 || vectorCode === null ? [] : vectorCode.getVectorMaintenanceStatus(manager);
|
|
7288
7332
|
},
|
|
7289
7333
|
on(event, handler) {
|
|
7290
7334
|
const key = event;
|
|
@@ -7326,4 +7370,4 @@ async function createNarsil(config) {
|
|
|
7326
7370
|
// src/index.ts
|
|
7327
7371
|
var VERSION = "0.1.0";
|
|
7328
7372
|
|
|
7329
|
-
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, bindRerankWindowSource, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineCoreOf, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, providedDocId, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, rerankWindowCapOf, resolveRunnableConfig, resolveVectorText, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, toError2 as toError, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|
|
7373
|
+
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, bindRerankWindowSource, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createMemoryMonitor, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineCoreOf, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, memoryFractionsOf, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, providedDocId, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, rerankWindowCapOf, resolveRunnableConfig, resolveVectorText, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, toError2 as toError, validateEmbeddingConfig, validateMemoryFractions, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|