@delali/narsil 0.3.1-next.144 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
- package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/chunk-3TVDTACC.mjs +104 -0
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
- package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/chunk-6BWLDLS3.mjs +58 -0
- package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
- package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/chunk-B7XIOAMK.mjs +100 -0
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
- package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
- package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
- package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
- package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
- package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
- package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/chunk-SOUZMPME.mjs +1635 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
- package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
- package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
- package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
- package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/chunk-YKEHQUSZ.mjs +248 -0
- package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +46 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
- package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16116 -14537
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +542 -8
- package/dist/index.mjs +29 -32
- package/dist/index.react-native.mjs +19 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
- package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
- package/dist/module-V5K4ZP23.mjs +45 -0
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +11 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +159 -38
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
- package/openapi.json +349 -2
- package/package.json +108 -8
- package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
- package/dist/module-IYPLWHGC.mjs +0 -48
- package/dist/thread-KGNAIKPD.mjs +0 -33
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -1,14 +1,18 @@
|
|
|
1
|
-
import { decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest,
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
1
|
+
import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-D6UFGBF5.mjs';
|
|
2
|
+
import './chunk-JNOAKDUM.mjs';
|
|
3
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-W643QV3L.mjs';
|
|
4
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-QMNZFFD3.mjs';
|
|
4
5
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
5
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-
|
|
6
|
-
import { encodeDocument,
|
|
6
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-KAEVGLYN.mjs';
|
|
7
|
+
import { encodeDocument, toError, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-BVX2MEHA.mjs';
|
|
8
|
+
import { fnv1a, createRequestId } from './chunk-MPK76GVS.mjs';
|
|
7
9
|
import './chunk-JPTQZWJQ.mjs';
|
|
8
10
|
import './chunk-26A5JFT2.mjs';
|
|
9
|
-
import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-
|
|
10
|
-
import
|
|
11
|
-
import {
|
|
11
|
+
import { turnsAreBudgeted, turnIsSpent, yieldTurn, compareCodePoints, waitOutsideCountedWrite, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-IAXZAVYL.mjs';
|
|
12
|
+
import './chunk-TY7L6OXO.mjs';
|
|
13
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-4RD3AFSB.mjs';
|
|
14
|
+
import './chunk-WXLBZJMT.mjs';
|
|
15
|
+
import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
|
|
12
16
|
import { decode } from '@msgpack/msgpack';
|
|
13
17
|
|
|
14
18
|
// src/persistence/durability/node-fs.browser.ts
|
|
@@ -37,7 +41,7 @@ async function removeStaleLock(_lockPath, _asidePath, _isStale) {
|
|
|
37
41
|
|
|
38
42
|
// src/persistence/durability/directory-lock.ts
|
|
39
43
|
var LOCK_FILE_NAME = ".narsil.lock";
|
|
40
|
-
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
44
|
+
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EACCES", "EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
41
45
|
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
42
46
|
function errnoOf(err) {
|
|
43
47
|
return err?.code;
|
|
@@ -365,9 +369,9 @@ function createDurableDirectory(root) {
|
|
|
365
369
|
if (stat.size === 0) {
|
|
366
370
|
return new Uint8Array(0);
|
|
367
371
|
}
|
|
368
|
-
const
|
|
369
|
-
await handle.read(
|
|
370
|
-
return
|
|
372
|
+
const bytes = new Uint8Array(stat.size);
|
|
373
|
+
await handle.read(bytes, 0, stat.size, 0);
|
|
374
|
+
return bytes;
|
|
371
375
|
},
|
|
372
376
|
async writeSlot(offset, slot) {
|
|
373
377
|
try {
|
|
@@ -825,7 +829,7 @@ function recoverDirectoryRestores(directory) {
|
|
|
825
829
|
}
|
|
826
830
|
|
|
827
831
|
// src/persistence/durability/restore-staging.ts
|
|
828
|
-
function
|
|
832
|
+
function toError2(error) {
|
|
829
833
|
return error instanceof Error ? error : new Error(String(error));
|
|
830
834
|
}
|
|
831
835
|
function pathUnder(path, root) {
|
|
@@ -894,7 +898,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
894
898
|
await storage.remove(journalKey(indexName));
|
|
895
899
|
} catch (removal) {
|
|
896
900
|
committed = true;
|
|
897
|
-
options.onStorageLost?.(
|
|
901
|
+
options.onStorageLost?.(toError2(removal));
|
|
898
902
|
}
|
|
899
903
|
throw error;
|
|
900
904
|
}
|
|
@@ -904,7 +908,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
|
|
|
904
908
|
try {
|
|
905
909
|
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
906
910
|
} catch (error) {
|
|
907
|
-
options.onStorageLost?.(
|
|
911
|
+
options.onStorageLost?.(toError2(error));
|
|
908
912
|
}
|
|
909
913
|
},
|
|
910
914
|
async removeRetired() {
|
|
@@ -936,7 +940,7 @@ function adapterRestoreStorage(adapter) {
|
|
|
936
940
|
}
|
|
937
941
|
|
|
938
942
|
// src/persistence/durability/archive-checkpoint.ts
|
|
939
|
-
function
|
|
943
|
+
function toError3(error) {
|
|
940
944
|
return error instanceof Error ? error : new Error(String(error));
|
|
941
945
|
}
|
|
942
946
|
function stageDirectoryRestore(deps, indexName) {
|
|
@@ -945,7 +949,7 @@ function stageDirectoryRestore(deps, indexName) {
|
|
|
945
949
|
try {
|
|
946
950
|
await run();
|
|
947
951
|
} catch (error) {
|
|
948
|
-
deps.markFatal(
|
|
952
|
+
deps.markFatal(toError3(error));
|
|
949
953
|
throw error;
|
|
950
954
|
}
|
|
951
955
|
},
|
|
@@ -969,8 +973,8 @@ function createArchiveCheckpointAccess(deps) {
|
|
|
969
973
|
deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
|
|
970
974
|
const release = releaser(deps.held, indexName);
|
|
971
975
|
try {
|
|
972
|
-
await deps.checkpoint(indexName);
|
|
973
|
-
await deps.checkpoint(indexName);
|
|
976
|
+
await deps.checkpoint(indexName, false);
|
|
977
|
+
await deps.checkpoint(indexName, true);
|
|
974
978
|
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
975
979
|
if (manifestBytes === null) {
|
|
976
980
|
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
@@ -1124,11 +1128,12 @@ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive,
|
|
|
1124
1128
|
}
|
|
1125
1129
|
let highestSeqNo = fromSeqNoExclusive;
|
|
1126
1130
|
let highestPrimaryTerm = 0;
|
|
1127
|
-
const replay = (entries) => {
|
|
1131
|
+
const replay = async (entries) => {
|
|
1128
1132
|
for (const entry of entries) {
|
|
1129
1133
|
if (entry.seqNo <= highestSeqNo) {
|
|
1130
1134
|
continue;
|
|
1131
1135
|
}
|
|
1136
|
+
if (turnIsSpent()) await yieldTurn();
|
|
1132
1137
|
applyEntry(entry, deps);
|
|
1133
1138
|
highestSeqNo = entry.seqNo;
|
|
1134
1139
|
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
@@ -1136,9 +1141,9 @@ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive,
|
|
|
1136
1141
|
}
|
|
1137
1142
|
}
|
|
1138
1143
|
};
|
|
1139
|
-
replay(durableEntries);
|
|
1144
|
+
await replay(durableEntries);
|
|
1140
1145
|
if (activeTail !== null) {
|
|
1141
|
-
replay(activeTail.entries);
|
|
1146
|
+
await replay(activeTail.entries);
|
|
1142
1147
|
}
|
|
1143
1148
|
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
1144
1149
|
if (activeTail !== null && activeTail.cleanEnd < activeTail.segmentLength) {
|
|
@@ -1171,7 +1176,7 @@ function applyEntry(entry, deps) {
|
|
|
1171
1176
|
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
1172
1177
|
return;
|
|
1173
1178
|
}
|
|
1174
|
-
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes);
|
|
1179
|
+
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
1175
1180
|
}
|
|
1176
1181
|
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
1177
1182
|
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
@@ -1266,7 +1271,7 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
1266
1271
|
return whole;
|
|
1267
1272
|
};
|
|
1268
1273
|
}
|
|
1269
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
1274
|
+
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
|
|
1270
1275
|
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
1271
1276
|
return whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
1272
1277
|
onCaptured?.();
|
|
@@ -1291,14 +1296,19 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
1291
1296
|
targets.push(target);
|
|
1292
1297
|
}
|
|
1293
1298
|
const vectorPlans = /* @__PURE__ */ new Map();
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1299
|
+
try {
|
|
1300
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
1301
|
+
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
1302
|
+
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
1303
|
+
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
|
|
1304
|
+
}
|
|
1305
|
+
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
1306
|
+
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
1307
|
+
return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
|
|
1308
|
+
} catch (error) {
|
|
1309
|
+
for (const plan of vectorPlans.values()) plan.release();
|
|
1310
|
+
throw error;
|
|
1311
|
+
}
|
|
1302
1312
|
});
|
|
1303
1313
|
}
|
|
1304
1314
|
async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
|
|
@@ -1319,6 +1329,15 @@ async function writeCapturedVectors(store, indexName, capture2, priorManifest, r
|
|
|
1319
1329
|
return { vectors: written.refs, layouts: written.layouts, written: written.written };
|
|
1320
1330
|
}
|
|
1321
1331
|
|
|
1332
|
+
// src/persistence/durability/checkpoint-worker-entry.ts
|
|
1333
|
+
function resolveCheckpointWorkerEntry() {
|
|
1334
|
+
return resolveWorkerEntry(
|
|
1335
|
+
import.meta.url,
|
|
1336
|
+
/\/src\/persistence\/durability\/[^/]+$/,
|
|
1337
|
+
"persistence/durability/checkpoint-worker.mjs"
|
|
1338
|
+
);
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1322
1341
|
// src/persistence/durability/checkpoint-worker-dispatch.ts
|
|
1323
1342
|
var workerUsable = true;
|
|
1324
1343
|
var pooledWorker = null;
|
|
@@ -1327,11 +1346,7 @@ var spawnedWorkerCount = 0;
|
|
|
1327
1346
|
var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
|
|
1328
1347
|
var idleTimer = null;
|
|
1329
1348
|
async function spawnWorker() {
|
|
1330
|
-
const entryPoint =
|
|
1331
|
-
import.meta.url,
|
|
1332
|
-
/\/src\/persistence\/durability\/[^/]+$/,
|
|
1333
|
-
"persistence/durability/checkpoint-worker.mjs"
|
|
1334
|
-
);
|
|
1349
|
+
const entryPoint = resolveCheckpointWorkerEntry();
|
|
1335
1350
|
if (entryPoint === null) return null;
|
|
1336
1351
|
try {
|
|
1337
1352
|
const worker = await spawnNodeWorker(new URL(entryPoint));
|
|
@@ -1562,9 +1577,11 @@ function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES
|
|
|
1562
1577
|
}
|
|
1563
1578
|
async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
|
|
1564
1579
|
while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
|
|
1565
|
-
await
|
|
1566
|
-
|
|
1567
|
-
|
|
1580
|
+
await waitOutsideCountedWrite(
|
|
1581
|
+
() => new Promise((resume) => {
|
|
1582
|
+
indexState.stalledWrites.push(resume);
|
|
1583
|
+
})
|
|
1584
|
+
);
|
|
1568
1585
|
}
|
|
1569
1586
|
}
|
|
1570
1587
|
function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
|
|
@@ -1646,7 +1663,8 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1646
1663
|
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
|
|
1647
1664
|
textOnDisk ? () => {
|
|
1648
1665
|
changesSinceCapture = manager.openCheckpointChangeLog();
|
|
1649
|
-
} : void 0
|
|
1666
|
+
} : void 0,
|
|
1667
|
+
input.pinsRetiredVectors
|
|
1650
1668
|
);
|
|
1651
1669
|
try {
|
|
1652
1670
|
await openNewLogSegments(input.indexState, input.failPartition);
|
|
@@ -1654,12 +1672,13 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1654
1672
|
return changesSinceCapture;
|
|
1655
1673
|
});
|
|
1656
1674
|
} finally {
|
|
1675
|
+
for (const plan of capture2.vectorPlans.values()) plan.release();
|
|
1657
1676
|
changesSinceCapture?.close();
|
|
1658
1677
|
}
|
|
1659
1678
|
}
|
|
1660
1679
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1661
1680
|
const { indexName, indexState, hooks } = input;
|
|
1662
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-
|
|
1681
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-CEKA7MQH.mjs');
|
|
1663
1682
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1664
1683
|
if (adoption === null) return [];
|
|
1665
1684
|
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
@@ -1714,7 +1733,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1714
1733
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1715
1734
|
if (!input.holdsCheckpointFiles()) {
|
|
1716
1735
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1717
|
-
const stillRead = metadata.textStorage === "disk" ? (await import('./module-
|
|
1736
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-CEKA7MQH.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1718
1737
|
await removeCheckpointGarbage(
|
|
1719
1738
|
store,
|
|
1720
1739
|
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
@@ -1756,7 +1775,10 @@ function recordsBeyond(indexState, targets) {
|
|
|
1756
1775
|
function createIndexLoads(vectorIndexesOf2) {
|
|
1757
1776
|
const loads = /* @__PURE__ */ new Map();
|
|
1758
1777
|
const startGraphRepairs = (indexName) => {
|
|
1759
|
-
for (const vectorIndex of vectorIndexesOf2(indexName).values())
|
|
1778
|
+
for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
|
|
1779
|
+
vectorIndex.scheduleBuild();
|
|
1780
|
+
vectorIndex.startShuffledRebuildWhenDue();
|
|
1781
|
+
}
|
|
1760
1782
|
};
|
|
1761
1783
|
return {
|
|
1762
1784
|
track(indexName, load, startsGraphRepairs = true) {
|
|
@@ -1957,8 +1979,28 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
1957
1979
|
}
|
|
1958
1980
|
}
|
|
1959
1981
|
|
|
1982
|
+
// src/persistence/durability/metadata-write-queue.ts
|
|
1983
|
+
function createMetadataWriteQueue() {
|
|
1984
|
+
const writes = /* @__PURE__ */ new Map();
|
|
1985
|
+
return {
|
|
1986
|
+
queue(indexName, write) {
|
|
1987
|
+
const previous = writes.get(indexName) ?? Promise.resolve();
|
|
1988
|
+
const queued = previous.then(write);
|
|
1989
|
+
writes.set(
|
|
1990
|
+
indexName,
|
|
1991
|
+
queued.catch(() => void 0)
|
|
1992
|
+
);
|
|
1993
|
+
return queued;
|
|
1994
|
+
},
|
|
1995
|
+
pending: (indexName) => writes.get(indexName),
|
|
1996
|
+
forget(indexName) {
|
|
1997
|
+
writes.delete(indexName);
|
|
1998
|
+
}
|
|
1999
|
+
};
|
|
2000
|
+
}
|
|
2001
|
+
|
|
1960
2002
|
// src/persistence/durability/mutation-batch.ts
|
|
1961
|
-
function
|
|
2003
|
+
function toError4(err) {
|
|
1962
2004
|
return err instanceof Error ? err : new Error(String(err));
|
|
1963
2005
|
}
|
|
1964
2006
|
function requireAscendingSeqNo(record, seqNo, previousSeqNo) {
|
|
@@ -1978,7 +2020,7 @@ function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
|
1978
2020
|
}
|
|
1979
2021
|
}
|
|
1980
2022
|
function failPartition(partition, address, deps, err) {
|
|
1981
|
-
const error =
|
|
2023
|
+
const error = toError4(err);
|
|
1982
2024
|
if (partition.failed !== null) return partition.failed;
|
|
1983
2025
|
partition.failed = error;
|
|
1984
2026
|
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
@@ -2039,6 +2081,7 @@ async function recordPartitionBatch(partition, records, deps) {
|
|
|
2039
2081
|
}
|
|
2040
2082
|
const applied = [];
|
|
2041
2083
|
for (let i = 0; i < records.length; i++) {
|
|
2084
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2042
2085
|
try {
|
|
2043
2086
|
await records[i].apply();
|
|
2044
2087
|
applied.push(i);
|
|
@@ -2053,6 +2096,7 @@ async function recordPartitionBatch(partition, records, deps) {
|
|
|
2053
2096
|
const entries = [];
|
|
2054
2097
|
try {
|
|
2055
2098
|
for (const i of applied) {
|
|
2099
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2056
2100
|
let numbered;
|
|
2057
2101
|
try {
|
|
2058
2102
|
numbered = deps.numberEntry(records[i]);
|
|
@@ -2090,7 +2134,7 @@ async function recordPartitionBatch(partition, records, deps) {
|
|
|
2090
2134
|
}
|
|
2091
2135
|
}
|
|
2092
2136
|
} catch (err) {
|
|
2093
|
-
failUnlessAlreadyFailed(outcomes, records.length,
|
|
2137
|
+
failUnlessAlreadyFailed(outcomes, records.length, toError4(err));
|
|
2094
2138
|
}
|
|
2095
2139
|
return outcomes;
|
|
2096
2140
|
}
|
|
@@ -2417,7 +2461,11 @@ function createWalWriter(directory, config) {
|
|
|
2417
2461
|
if (entries.length === 0) {
|
|
2418
2462
|
return;
|
|
2419
2463
|
}
|
|
2420
|
-
const frames =
|
|
2464
|
+
const frames = [];
|
|
2465
|
+
for (const entry of entries) {
|
|
2466
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2467
|
+
frames.push(frameRecord(entry));
|
|
2468
|
+
}
|
|
2421
2469
|
await ensureSegment(entries[0].seqNo);
|
|
2422
2470
|
let start = 0;
|
|
2423
2471
|
while (start < frames.length) {
|
|
@@ -2929,7 +2977,7 @@ var decoder = new TextDecoder();
|
|
|
2929
2977
|
async function countPagedPartition(store, partition) {
|
|
2930
2978
|
const { directory } = store;
|
|
2931
2979
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2932
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-
|
|
2980
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-CEKA7MQH.mjs');
|
|
2933
2981
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2934
2982
|
const scans = [];
|
|
2935
2983
|
try {
|
|
@@ -3310,7 +3358,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3310
3358
|
manager,
|
|
3311
3359
|
vectorFieldPaths: hooks.getVectorFieldPaths(indexName),
|
|
3312
3360
|
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
3313
|
-
writeRecords
|
|
3361
|
+
writeRecords,
|
|
3362
|
+
defersGraphBuilds: true
|
|
3314
3363
|
};
|
|
3315
3364
|
const replay = async (checkpoint) => {
|
|
3316
3365
|
const replayed = [];
|
|
@@ -3376,7 +3425,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3376
3425
|
let commitDeferral = null;
|
|
3377
3426
|
const partitionFailures = createPartitionFailureRouting();
|
|
3378
3427
|
const heldCheckpoints = createHeldCheckpoints(hooks);
|
|
3379
|
-
const metadataWrites =
|
|
3428
|
+
const metadataWrites = createMetadataWriteQueue();
|
|
3429
|
+
const queueMetadataWrite = metadataWrites.queue;
|
|
3380
3430
|
let checkpointTimer = null;
|
|
3381
3431
|
let asyncFlushTimer = null;
|
|
3382
3432
|
let shuttingDown = false;
|
|
@@ -3397,25 +3447,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3397
3447
|
stopTimers();
|
|
3398
3448
|
hooks.onFatalError(error);
|
|
3399
3449
|
}
|
|
3400
|
-
function toError5(err) {
|
|
3401
|
-
return err instanceof Error ? err : new Error(String(err));
|
|
3402
|
-
}
|
|
3403
3450
|
async function closeWriterReportingFailure(closePromise) {
|
|
3404
3451
|
try {
|
|
3405
3452
|
await closePromise;
|
|
3406
3453
|
} catch (err) {
|
|
3407
|
-
markFatal(
|
|
3454
|
+
markFatal(toError(err));
|
|
3408
3455
|
}
|
|
3409
3456
|
}
|
|
3410
|
-
function queueMetadataWrite(indexName, write) {
|
|
3411
|
-
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
3412
|
-
const queued = previous.then(write);
|
|
3413
|
-
metadataWrites.set(
|
|
3414
|
-
indexName,
|
|
3415
|
-
queued.catch(() => void 0)
|
|
3416
|
-
);
|
|
3417
|
-
return queued;
|
|
3418
|
-
}
|
|
3419
3457
|
function startCheckpointTimer() {
|
|
3420
3458
|
if (checkpointTimer !== null || shuttingDown || fatalError !== null || checkpointIntervalMs <= 0) {
|
|
3421
3459
|
return;
|
|
@@ -3439,25 +3477,25 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3439
3477
|
try {
|
|
3440
3478
|
await checkpointIndex(indexName);
|
|
3441
3479
|
} catch (err) {
|
|
3442
|
-
markFatal(
|
|
3480
|
+
markFatal(toError(err));
|
|
3443
3481
|
}
|
|
3444
3482
|
}
|
|
3445
3483
|
}
|
|
3446
|
-
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
|
|
3484
|
+
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false) {
|
|
3447
3485
|
const indexState = indexes.get(indexName);
|
|
3448
3486
|
if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
|
|
3449
3487
|
return checkpointInTurn(indexState, {
|
|
3450
|
-
joinsInFlight: !afterInFlight,
|
|
3488
|
+
joinsInFlight: !afterInFlight && !pinsRetiredVectors,
|
|
3451
3489
|
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
3452
|
-
perform: () => performCheckpoint(indexName, indexState, fromMemory),
|
|
3490
|
+
perform: () => performCheckpoint(indexName, indexState, fromMemory, pinsRetiredVectors),
|
|
3453
3491
|
afterSettling: () => releaseStalledWrites(indexState)
|
|
3454
3492
|
});
|
|
3455
3493
|
}
|
|
3456
|
-
async function requestedCheckpoint(indexName, fromMemory = false) {
|
|
3494
|
+
async function requestedCheckpoint(indexName, fromMemory = false, pinsVectors = false) {
|
|
3457
3495
|
heldCheckpoints.refuse(indexName, indexes.get(indexName));
|
|
3458
|
-
return checkpointIndex(indexName, fromMemory);
|
|
3496
|
+
return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors);
|
|
3459
3497
|
}
|
|
3460
|
-
async function performCheckpoint(indexName, indexState, fromMemory = false) {
|
|
3498
|
+
async function performCheckpoint(indexName, indexState, fromMemory = false, pinsRetiredVectors = false) {
|
|
3461
3499
|
await directoryWatch.verify();
|
|
3462
3500
|
if (fatalError !== null) throw fatalError;
|
|
3463
3501
|
await runDurableCheckpoint({
|
|
@@ -3468,6 +3506,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3468
3506
|
compactionThreshold,
|
|
3469
3507
|
canOffload: canOffloadCheckpoint,
|
|
3470
3508
|
fromMemory,
|
|
3509
|
+
pinsRetiredVectors,
|
|
3471
3510
|
mutationThreshold: checkpointMutationThreshold,
|
|
3472
3511
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
3473
3512
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
@@ -3509,7 +3548,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3509
3548
|
indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
|
|
3510
3549
|
if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
|
|
3511
3550
|
void checkpointIndex(indexName).catch((err) => {
|
|
3512
|
-
markFatal(
|
|
3551
|
+
markFatal(toError(err));
|
|
3513
3552
|
});
|
|
3514
3553
|
}
|
|
3515
3554
|
await stallWhileCheckpointsFallBehind(indexState);
|
|
@@ -3613,7 +3652,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3613
3652
|
},
|
|
3614
3653
|
async checkpointAfterInFlight(indexName) {
|
|
3615
3654
|
await loads.settled(indexName);
|
|
3616
|
-
await checkpointIndex(indexName, false, true).catch((err) => markFatal(
|
|
3655
|
+
await checkpointIndex(indexName, false, true).catch((err) => markFatal(toError(err)));
|
|
3617
3656
|
},
|
|
3618
3657
|
async checkpointAll() {
|
|
3619
3658
|
for (const indexName of [...indexes.keys()]) {
|
|
@@ -3624,9 +3663,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3624
3663
|
hooks.writeRecords?.dropIndex(indexName);
|
|
3625
3664
|
const indexState = indexes.get(indexName);
|
|
3626
3665
|
if (indexState !== void 0) {
|
|
3627
|
-
await drainIndexStateForUnload(indexState, metadataWrites.
|
|
3666
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
|
|
3628
3667
|
indexes.delete(indexName);
|
|
3629
|
-
metadataWrites.
|
|
3668
|
+
metadataWrites.forget(indexName);
|
|
3630
3669
|
}
|
|
3631
3670
|
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
3632
3671
|
for (const key of await directory.list(`${indexName}/`)) {
|
|
@@ -3638,14 +3677,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3638
3677
|
if (indexState === void 0) {
|
|
3639
3678
|
return;
|
|
3640
3679
|
}
|
|
3641
|
-
await drainIndexStateForUnload(indexState, metadataWrites.
|
|
3680
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
3642
3681
|
indexes.delete(indexName);
|
|
3643
|
-
metadataWrites.
|
|
3682
|
+
metadataWrites.forget(indexName);
|
|
3644
3683
|
},
|
|
3645
3684
|
...createArchiveCheckpointAccess({
|
|
3646
3685
|
directory,
|
|
3647
3686
|
held: heldCheckpointFiles,
|
|
3648
|
-
checkpoint: (name) => requestedCheckpoint(name),
|
|
3687
|
+
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3649
3688
|
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
3650
3689
|
markFatal
|
|
3651
3690
|
}),
|
|
@@ -3735,7 +3774,7 @@ function createSnapshotIndexState() {
|
|
|
3735
3774
|
}
|
|
3736
3775
|
|
|
3737
3776
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
3738
|
-
function
|
|
3777
|
+
function toError5(error) {
|
|
3739
3778
|
return error instanceof Error ? error : new Error(String(error));
|
|
3740
3779
|
}
|
|
3741
3780
|
function createPendingRestores(onFatalError) {
|
|
@@ -3756,7 +3795,7 @@ function createPendingRestores(onFatalError) {
|
|
|
3756
3795
|
return running;
|
|
3757
3796
|
};
|
|
3758
3797
|
pending.set(indexName, finish);
|
|
3759
|
-
void finish().catch((error) => onFatalError(
|
|
3798
|
+
void finish().catch((error) => onFatalError(toError5(error)));
|
|
3760
3799
|
return Promise.resolve();
|
|
3761
3800
|
},
|
|
3762
3801
|
async settle(indexName) {
|
|
@@ -4142,6 +4181,7 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4142
4181
|
const outcomes = new Array(mutations.length);
|
|
4143
4182
|
const recordPositions = [];
|
|
4144
4183
|
for (let i = 0; i < mutations.length; i++) {
|
|
4184
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
4145
4185
|
const mutation = mutations[i];
|
|
4146
4186
|
try {
|
|
4147
4187
|
records.push({
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { gunzip, gzip } from './chunk-JNOAKDUM.mjs';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-2JABMB4S.mjs';
|
|
2
|
+
import './chunk-NVEJJP4R.mjs';
|
|
3
|
+
import './chunk-5V74FFZE.mjs';
|
|
4
|
+
import './chunk-BVX2MEHA.mjs';
|
|
5
|
+
import './chunk-MPK76GVS.mjs';
|
|
6
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
7
|
+
import './chunk-26A5JFT2.mjs';
|
|
8
|
+
import './chunk-IAXZAVYL.mjs';
|
|
9
|
+
import './chunk-TY7L6OXO.mjs';
|
|
10
|
+
import './chunk-4RD3AFSB.mjs';
|
|
11
|
+
import './chunk-WXLBZJMT.mjs';
|
|
12
|
+
import './chunk-PIS7ZAP7.mjs';
|
|
@@ -1,12 +1,15 @@
|
|
|
1
|
-
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
1
|
+
export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-XILIT5MT.mjs';
|
|
2
|
+
import './chunk-6OWX3OP2.mjs';
|
|
3
|
+
import './chunk-W643QV3L.mjs';
|
|
4
|
+
import './chunk-QMNZFFD3.mjs';
|
|
5
|
+
import './chunk-5V74FFZE.mjs';
|
|
6
|
+
import './chunk-LURXVCHG.mjs';
|
|
7
|
+
import './chunk-BVX2MEHA.mjs';
|
|
8
|
+
import './chunk-MPK76GVS.mjs';
|
|
8
9
|
import './chunk-JPTQZWJQ.mjs';
|
|
9
10
|
import './chunk-26A5JFT2.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-
|
|
11
|
+
import './chunk-IAXZAVYL.mjs';
|
|
12
|
+
import './chunk-TY7L6OXO.mjs';
|
|
13
|
+
import './chunk-4RD3AFSB.mjs';
|
|
14
|
+
import './chunk-WXLBZJMT.mjs';
|
|
15
|
+
import './chunk-PIS7ZAP7.mjs';
|