@delali/narsil 0.4.1 → 0.4.2-next.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
- package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
- package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
- package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
- package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
- package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
- package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
- package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
- package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
- package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
- package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
- package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
- package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
- package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
- package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
- package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
- package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
- package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
- package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -12
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -1,48 +1,49 @@
|
|
|
1
|
-
import { scheduleSyncRetention, createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-
|
|
2
|
-
export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-
|
|
3
|
-
import { buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, documentExportStream, requireRestoreMemory, memoryCheckpointStore, admitDurableWrite, metadataConfigOf, durableMetadataConfigOf, unpackIndexSnapshotEnvelope, nestedSchemaOf, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, loadSegmentedCheckpoint, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, requireSavedPartitionCount, createDetachedIndex, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn,
|
|
4
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, changesOf, createByteReader, documentExportStream, dropHeldAnalysisChange, executeRebalance, followIndexChanges, heldAnalysisIsBuilt, holdsNamedSettings, openFileVectorReads, pagedPartitionLines, readDocumentExport, releaseHeldAnalysisChange, requireSavedPartitionCount, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout, updateChangesOf } from './chunk-
|
|
5
|
-
import { readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-
|
|
6
|
-
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-
|
|
7
|
-
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, indexConfigurationOf, updateIndexSync, decodeIndexConfiguration } from './chunk-
|
|
8
|
-
export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, persistEpochStates, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-
|
|
1
|
+
import { scheduleSyncRetention, createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-QNZJYMCF.mjs';
|
|
2
|
+
export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-QNZJYMCF.mjs';
|
|
3
|
+
import { buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, documentExportStream, requireRestoreMemory, memoryCheckpointStore, admitDurableWrite, metadataConfigOf, durableMetadataConfigOf, unpackIndexSnapshotEnvelope, nestedSchemaOf, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, loadSegmentedCheckpoint, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, commitAppended, listPersistedIndexes, removeLeftoverRewriteFiles, requireSavedPartitionCount, createDetachedIndex, shutDownRefusal, recordBatchesByPartition, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, partitionFailureFor, SINGLE_NODE_PRIMARY_TERM, deserializePayloadV1, readCurrentManifest, drainIndexStateForUnload, readSegmentManifest, readMetadataEnvelope, snapshotCheckpointFor, whileGraphsStayComplete, captureAppliedPrefix, createSeqOwner, commitEveryPartition, loadMetadata, reclaimUnlistedKeys, writeLiveVectors, dropStoredVectorValues, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, highestPartitionIdOf, growPartitionsTo, createMarkerWriter, metadataFromConfig, metadataPartitionCount } from './chunk-BWPVDIRN.mjs';
|
|
4
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, changesOf, createByteReader, documentExportStream, dropHeldAnalysisChange, executeRebalance, followIndexChanges, heldAnalysisIsBuilt, holdsNamedSettings, openFileVectorReads, pagedPartitionLines, readDocumentExport, releaseHeldAnalysisChange, requireSavedPartitionCount, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout, updateChangesOf } from './chunk-BWPVDIRN.mjs';
|
|
5
|
+
import { readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-FE2XILWN.mjs';
|
|
6
|
+
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-FE2XILWN.mjs';
|
|
7
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, indexConfigurationOf, updateIndexSync, decodeIndexConfiguration } from './chunk-PNFP5THV.mjs';
|
|
8
|
+
export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, persistEpochStates, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-PNFP5THV.mjs';
|
|
9
9
|
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-M2Z5WVWW.mjs';
|
|
10
|
-
import { readSnapshotArchive, decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-
|
|
11
|
-
export { readSnapshotArchive } from './chunk-
|
|
12
|
-
import { createDurableDirectory } from './chunk-
|
|
13
|
-
import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, bytesThatSegmentsHoldInMemory, savedBytesOfManifest, checkPagedSegmentFully, readSegmentFile, NOTHING_SAVED, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, checkpointGarbage, removeCheckpointGarbage, nextSegmentKey, rewrittenHeaderKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, openPagedSegmentScan, countsAsLatestHolder, segmentKey, WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT, SEGMENT_TAIL_PATTERN, writeSegmentHeader, WAL_FORMAT_VERSION, parseSegmentStartSeqNo, readWalSegments, joinFrames
|
|
14
|
-
export { bytesThatSegmentsHoldInMemory, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSegmentScan, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-
|
|
15
|
-
import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-
|
|
10
|
+
import { readSnapshotArchive, decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-LRIJ7U3T.mjs';
|
|
11
|
+
export { readSnapshotArchive } from './chunk-LRIJ7U3T.mjs';
|
|
12
|
+
import { createDurableDirectory } from './chunk-KL4ORYJW.mjs';
|
|
13
|
+
import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, bytesThatSegmentsHoldInMemory, savedBytesOfManifest, checkPagedSegmentFully, readSegmentFile, buildEntry, NOTHING_SAVED, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, checkpointGarbage, removeCheckpointGarbage, nextSegmentKey, rewrittenHeaderKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, openPagedSegmentScan, countsAsLatestHolder, segmentKey, WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT, SEGMENT_TAIL_PATTERN, writeSegmentHeader, WAL_FORMAT_VERSION, parseSegmentStartSeqNo, readWalSegments, joinFrames } from './chunk-6SIEVWMG.mjs';
|
|
14
|
+
export { bytesThatSegmentsHoldInMemory, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSegmentScan, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-6SIEVWMG.mjs';
|
|
15
|
+
import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-H23FB267.mjs';
|
|
16
16
|
import './chunk-5FLRXF73.mjs';
|
|
17
17
|
import './chunk-JDDXV6UF.mjs';
|
|
18
|
-
import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-
|
|
19
|
-
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
20
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-MSIN4NM3.mjs';
|
|
18
|
+
import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-GTGQLCEI.mjs';
|
|
21
19
|
import { encode } from './chunk-S5QMWECL.mjs';
|
|
22
20
|
export { bindStorageHost } from './chunk-S5QMWECL.mjs';
|
|
23
21
|
import { sha256 } from './chunk-LNGE7SLY.mjs';
|
|
24
|
-
import { setNestedValue } from './chunk-
|
|
25
|
-
import { rewriteSpillOf } from './chunk-
|
|
26
|
-
import { deserializePartition } from './chunk-
|
|
22
|
+
import { setNestedValue } from './chunk-5SHWA6PE.mjs';
|
|
23
|
+
import { rewriteSpillOf } from './chunk-PQWK7KXS.mjs';
|
|
24
|
+
import { deserializePartition } from './chunk-ZO7VLAIT.mjs';
|
|
27
25
|
import { decodeVectorIndexParts, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, graphPartsOfEarlierPayload, decodeVectorGraphPayload } from './chunk-R724DJSG.mjs';
|
|
28
26
|
export { createHNSWIndex, osqBitsOf } from './chunk-R724DJSG.mjs';
|
|
29
27
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
30
|
-
import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-
|
|
31
|
-
import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, ARRAY_POSITION_GAP, fnv1a, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, toError, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createPartitionRouter, buildSegmentPayloads, createRequestId } from './chunk-
|
|
28
|
+
import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-G3KVEHIW.mjs';
|
|
29
|
+
import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, ARRAY_POSITION_GAP, fnv1a, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, toError, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createPartitionRouter, buildSegmentPayloads, createRequestId } from './chunk-WYH3QHBK.mjs';
|
|
32
30
|
import { encodeDocument } from './chunk-2CKDCC65.mjs';
|
|
33
31
|
import './chunk-2AFZE7Y3.mjs';
|
|
34
|
-
import { decode } from './chunk-
|
|
32
|
+
import { decode } from './chunk-Y7GXGJFZ.mjs';
|
|
35
33
|
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
36
34
|
import './chunk-7T4LAYT4.mjs';
|
|
37
|
-
import './chunk-
|
|
38
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, parseVectorDimension, getNestedValue, requireValidFieldTypes } from './chunk-
|
|
35
|
+
import './chunk-4JLHHEBN.mjs';
|
|
36
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, parseVectorDimension, getNestedValue, requireValidFieldTypes } from './chunk-EY5U4P5V.mjs';
|
|
39
37
|
import './chunk-U2BKYEC7.mjs';
|
|
40
38
|
import { detectRuntime, spawnNodeWorker, resolveWorkerEntry } from './chunk-FJXJZUWF.mjs';
|
|
41
39
|
import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-5EI5CXO6.mjs';
|
|
42
40
|
import { STATISTICS_BROADCAST_INTERVAL_MS, FOREIGN_STATS_TTL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-3ECIRFS5.mjs';
|
|
43
41
|
import './chunk-ASZLNVWW.mjs';
|
|
44
42
|
import { MIN_M } from './chunk-ZXX5GZEX.mjs';
|
|
43
|
+
import { getFs, renameUnlessMissing } from './chunk-ZYEEOZTT.mjs';
|
|
44
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, INDEXEDDB_BLOCK_BYTES, INDEXEDDB_CACHED_BLOCKS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-S3JHYPH7.mjs';
|
|
45
45
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
46
|
+
import { INDEXEDDB_BLOCKED_TIMEOUT_MS } from './chunk-K5HSUPYS.mjs';
|
|
46
47
|
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
47
48
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
48
49
|
|
|
@@ -1272,27 +1273,27 @@ function whileNoMutationApplies(partitions, capture2) {
|
|
|
1272
1273
|
}
|
|
1273
1274
|
});
|
|
1274
1275
|
}
|
|
1275
|
-
async function makeEveryAppliedMutationDurable(indexState,
|
|
1276
|
+
async function makeEveryAppliedMutationDurable(indexState, failPartition) {
|
|
1276
1277
|
for (const [partitionId, partition] of [...indexState.partitions]) {
|
|
1277
1278
|
await partition.appendChain;
|
|
1278
1279
|
if (partition.failed !== null) continue;
|
|
1279
1280
|
try {
|
|
1280
1281
|
await partition.walWriter.commit();
|
|
1281
1282
|
} catch (error) {
|
|
1282
|
-
|
|
1283
|
+
failPartition(partitionId, partition, error);
|
|
1283
1284
|
}
|
|
1284
1285
|
}
|
|
1285
1286
|
}
|
|
1286
|
-
async function openNewLogSegments(indexState,
|
|
1287
|
+
async function openNewLogSegments(indexState, failPartition) {
|
|
1287
1288
|
const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
|
|
1288
1289
|
try {
|
|
1289
1290
|
if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
|
|
1290
1291
|
} catch (error) {
|
|
1291
|
-
|
|
1292
|
+
failPartition(partitionId, partition, error);
|
|
1292
1293
|
return;
|
|
1293
1294
|
}
|
|
1294
1295
|
const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
|
|
1295
|
-
|
|
1296
|
+
failPartition(partitionId, partition, error);
|
|
1296
1297
|
});
|
|
1297
1298
|
partition.appendChain = rolled;
|
|
1298
1299
|
await rolled;
|
|
@@ -2172,10 +2173,10 @@ function createLogBytesLedger(base, sizes) {
|
|
|
2172
2173
|
const moved = untrack(from);
|
|
2173
2174
|
if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
|
|
2174
2175
|
},
|
|
2175
|
-
list: (prefix) => base.list(prefix)
|
|
2176
|
-
pathOf: (key) => base.pathOf(key)
|
|
2176
|
+
list: (prefix) => base.list(prefix)
|
|
2177
2177
|
};
|
|
2178
|
-
const { readRange, byteLength, identity, lock } = base;
|
|
2178
|
+
const { readRange, byteLength, identity, lock, pathOf } = base;
|
|
2179
|
+
if (pathOf !== void 0) directory.pathOf = (key) => pathOf.call(base, key);
|
|
2179
2180
|
if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
|
|
2180
2181
|
if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
|
|
2181
2182
|
if (identity !== void 0) directory.identity = () => identity.call(base);
|
|
@@ -2345,21 +2346,27 @@ function storageFailed(code, message, key, error) {
|
|
|
2345
2346
|
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
2346
2347
|
}
|
|
2347
2348
|
function directoryRestoreStorage(directory) {
|
|
2348
|
-
|
|
2349
|
+
const storage = {
|
|
2349
2350
|
directory,
|
|
2350
2351
|
read: (key) => directory.read(key),
|
|
2351
2352
|
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
2352
2353
|
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
2353
2354
|
remove: (key) => directory.remove(key),
|
|
2354
2355
|
list: (prefix) => directory.list(prefix),
|
|
2355
|
-
moveKey: (from, to) => directory.move(from, to)
|
|
2356
|
-
|
|
2356
|
+
moveKey: (from, to) => directory.move(from, to)
|
|
2357
|
+
};
|
|
2358
|
+
const { pathOf } = directory;
|
|
2359
|
+
if (pathOf === void 0) return storage;
|
|
2360
|
+
return { ...storage, ...fileTreeOperations(directory, (key) => pathOf.call(directory, key)) };
|
|
2361
|
+
}
|
|
2362
|
+
function fileTreeOperations(directory, pathOf) {
|
|
2363
|
+
return {
|
|
2364
|
+
pathOf,
|
|
2357
2365
|
async moveTree(from, to) {
|
|
2358
|
-
const source = await
|
|
2359
|
-
const target = await
|
|
2360
|
-
const fs = await getFs();
|
|
2366
|
+
const source = await pathOf(from);
|
|
2367
|
+
const target = await pathOf(to);
|
|
2361
2368
|
try {
|
|
2362
|
-
await
|
|
2369
|
+
await renameUnlessMissing(source, target);
|
|
2363
2370
|
} catch (error) {
|
|
2364
2371
|
if (missing(error)) return;
|
|
2365
2372
|
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
@@ -2370,7 +2377,7 @@ function directoryRestoreStorage(directory) {
|
|
|
2370
2377
|
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
2371
2378
|
const fs = await getFs();
|
|
2372
2379
|
try {
|
|
2373
|
-
await fs.rm(await
|
|
2380
|
+
await fs.rm(await pathOf(key), { recursive: true, force: true });
|
|
2374
2381
|
} catch (error) {
|
|
2375
2382
|
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
2376
2383
|
}
|
|
@@ -2392,7 +2399,7 @@ function pathUnder(path, root) {
|
|
|
2392
2399
|
return next === "" || next === "/" || next === "\\";
|
|
2393
2400
|
}
|
|
2394
2401
|
function stagedDirectory(directory, staged, unstaged) {
|
|
2395
|
-
const { readRange, byteLength } = directory;
|
|
2402
|
+
const { readRange, byteLength, pathOf } = directory;
|
|
2396
2403
|
return {
|
|
2397
2404
|
root: directory.root,
|
|
2398
2405
|
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
@@ -2403,7 +2410,7 @@ function stagedDirectory(directory, staged, unstaged) {
|
|
|
2403
2410
|
remove: (key) => directory.remove(staged(key)),
|
|
2404
2411
|
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
2405
2412
|
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
2406
|
-
pathOf: (key) =>
|
|
2413
|
+
...pathOf === void 0 ? {} : { pathOf: (key) => pathOf.call(directory, staged(key)) },
|
|
2407
2414
|
...readRange === void 0 ? {} : {
|
|
2408
2415
|
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
2409
2416
|
},
|
|
@@ -3155,6 +3162,18 @@ function createIndexStorageAccess(deps) {
|
|
|
3155
3162
|
}
|
|
3156
3163
|
|
|
3157
3164
|
// src/persistence/durability/metadata-write-queue.ts
|
|
3165
|
+
async function writeIndexMetadataFile(directory, indexName, bytes) {
|
|
3166
|
+
try {
|
|
3167
|
+
await directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
3168
|
+
} catch (error) {
|
|
3169
|
+
if (error instanceof NarsilError) throw error;
|
|
3170
|
+
throw new NarsilError(
|
|
3171
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3172
|
+
`The engine cannot write the metadata of index "${indexName}" to the durability directory "${directory.root}"`,
|
|
3173
|
+
{ indexName, directory: directory.root, cause: error instanceof Error ? error.message : String(error) }
|
|
3174
|
+
);
|
|
3175
|
+
}
|
|
3176
|
+
}
|
|
3158
3177
|
function createMetadataWriteQueue() {
|
|
3159
3178
|
const writes = /* @__PURE__ */ new Map();
|
|
3160
3179
|
return {
|
|
@@ -3170,158 +3189,16 @@ function createMetadataWriteQueue() {
|
|
|
3170
3189
|
pending: (indexName) => writes.get(indexName),
|
|
3171
3190
|
forget(indexName) {
|
|
3172
3191
|
writes.delete(indexName);
|
|
3173
|
-
}
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
}
|
|
3181
|
-
function requireAscendingSeqNo(record, seqNo, previousSeqNo) {
|
|
3182
|
-
if (seqNo > previousSeqNo) return;
|
|
3183
|
-
throw new NarsilError(
|
|
3184
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3185
|
-
`Log record ${seqNo} for partition ${record.partitionId} of index "${record.indexName}" does not follow record ${previousSeqNo}`,
|
|
3186
|
-
{ indexName: record.indexName, partitionId: record.partitionId, seqNo, previousSeqNo }
|
|
3187
|
-
);
|
|
3188
|
-
}
|
|
3189
|
-
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
3190
|
-
for (let i = 0; i < count; i++) {
|
|
3191
|
-
const outcome = outcomes[i];
|
|
3192
|
-
if (outcome === void 0 || outcome.ok) {
|
|
3193
|
-
outcomes[i] = { ok: false, error };
|
|
3194
|
-
}
|
|
3195
|
-
}
|
|
3196
|
-
}
|
|
3197
|
-
function failPartition(partition, address, deps, err) {
|
|
3198
|
-
const error = toError4(err);
|
|
3199
|
-
if (partition.failed !== null) return partition.failed;
|
|
3200
|
-
partition.failed = error;
|
|
3201
|
-
if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
|
|
3202
|
-
return error;
|
|
3203
|
-
}
|
|
3204
|
-
async function commitEveryPartition(indexes, deps) {
|
|
3205
|
-
for (const [indexName, indexState] of indexes) {
|
|
3206
|
-
for (const [partitionId, partition] of indexState.partitions) {
|
|
3207
|
-
if (partition.failed !== null || deps.fatalError() !== null) continue;
|
|
3208
|
-
try {
|
|
3209
|
-
await partition.walWriter.commit();
|
|
3210
|
-
} catch (err) {
|
|
3211
|
-
failPartition(partition, { indexName, partitionId }, deps, err);
|
|
3212
|
-
}
|
|
3213
|
-
}
|
|
3214
|
-
}
|
|
3215
|
-
}
|
|
3216
|
-
function partitionFailureFor(indexName, deps) {
|
|
3217
|
-
return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
|
|
3218
|
-
}
|
|
3219
|
-
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
3220
|
-
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
3221
|
-
for (let i = 0; i < records.length; i++) {
|
|
3222
|
-
const partition = partitionOf(records[i]);
|
|
3223
|
-
const positions = positionsByPartition.get(partition);
|
|
3224
|
-
if (positions === void 0) {
|
|
3225
|
-
positionsByPartition.set(partition, [i]);
|
|
3226
|
-
} else {
|
|
3227
|
-
positions.push(i);
|
|
3228
|
-
}
|
|
3229
|
-
}
|
|
3230
|
-
const outcomes = new Array(records.length);
|
|
3231
|
-
await Promise.all(
|
|
3232
|
-
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
3233
|
-
const partitionOutcomes = await recordPartitionBatch(
|
|
3234
|
-
partition,
|
|
3235
|
-
positions.map((position) => records[position]),
|
|
3236
|
-
deps
|
|
3237
|
-
);
|
|
3238
|
-
for (let i = 0; i < positions.length; i++) {
|
|
3239
|
-
outcomes[positions[i]] = partitionOutcomes[i];
|
|
3240
|
-
}
|
|
3241
|
-
})
|
|
3242
|
-
);
|
|
3243
|
-
return outcomes;
|
|
3244
|
-
}
|
|
3245
|
-
async function recordPartitionBatch(partition, records, deps) {
|
|
3246
|
-
const outcomes = new Array(records.length);
|
|
3247
|
-
let appendedCount = 0;
|
|
3248
|
-
let commitsHere = false;
|
|
3249
|
-
const appended = partition.appendChain.then(async () => {
|
|
3250
|
-
const fatal = deps.fatalError();
|
|
3251
|
-
if (fatal !== null) {
|
|
3252
|
-
throw fatal;
|
|
3253
|
-
}
|
|
3254
|
-
if (partition.failed !== null) {
|
|
3255
|
-
throw partition.failed;
|
|
3256
|
-
}
|
|
3257
|
-
const applied = [];
|
|
3258
|
-
for (let i = 0; i < records.length; i++) {
|
|
3259
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3260
|
-
try {
|
|
3261
|
-
await records[i].apply();
|
|
3262
|
-
applied.push(i);
|
|
3263
|
-
} catch (err) {
|
|
3264
|
-
outcomes[i] = { ok: false, error: err };
|
|
3265
|
-
}
|
|
3266
|
-
}
|
|
3267
|
-
if (applied.length === 0) {
|
|
3268
|
-
return;
|
|
3269
|
-
}
|
|
3270
|
-
let lastSeqNo = partition.appliedSeqNo;
|
|
3271
|
-
const entries = [];
|
|
3272
|
-
try {
|
|
3273
|
-
for (const i of applied) {
|
|
3274
|
-
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3275
|
-
let numbered;
|
|
3276
|
-
try {
|
|
3277
|
-
numbered = deps.numberEntry(records[i]);
|
|
3278
|
-
} catch (err) {
|
|
3279
|
-
outcomes[i] = { ok: false, error: err };
|
|
3280
|
-
continue;
|
|
3281
|
-
}
|
|
3282
|
-
if (numbered === null) {
|
|
3283
|
-
lastSeqNo = partition.seqOwner.next();
|
|
3284
|
-
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
3285
|
-
} else {
|
|
3286
|
-
requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
|
|
3287
|
-
partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
|
|
3288
|
-
lastSeqNo = numbered.seqNo;
|
|
3289
|
-
entries.push(numbered);
|
|
3290
|
-
}
|
|
3291
|
-
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
3292
|
-
if (!deps.defersCommit(records[i])) commitsHere = true;
|
|
3293
|
-
}
|
|
3294
|
-
if (entries.length > 0) await partition.walWriter.appendAll(entries);
|
|
3295
|
-
} catch (err) {
|
|
3296
|
-
throw failPartition(partition, records[0], deps, err);
|
|
3297
|
-
}
|
|
3298
|
-
partition.appliedSeqNo = lastSeqNo;
|
|
3299
|
-
appendedCount = entries.length;
|
|
3300
|
-
});
|
|
3301
|
-
partition.appendChain = appended.catch(() => void 0);
|
|
3302
|
-
try {
|
|
3303
|
-
await appended;
|
|
3304
|
-
if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
|
|
3305
|
-
try {
|
|
3306
|
-
await partition.walWriter.commit();
|
|
3307
|
-
} catch (err) {
|
|
3308
|
-
throw failPartition(partition, records[0], deps, err);
|
|
3192
|
+
},
|
|
3193
|
+
async settleAll() {
|
|
3194
|
+
let waiting = [...writes.values()];
|
|
3195
|
+
while (waiting.length > 0) {
|
|
3196
|
+
await Promise.all(waiting);
|
|
3197
|
+
const settled = waiting;
|
|
3198
|
+
waiting = [...writes.values()].filter((write) => !settled.includes(write));
|
|
3309
3199
|
}
|
|
3310
3200
|
}
|
|
3311
|
-
}
|
|
3312
|
-
failUnlessAlreadyFailed(outcomes, records.length, toError4(err));
|
|
3313
|
-
}
|
|
3314
|
-
return outcomes;
|
|
3315
|
-
}
|
|
3316
|
-
async function commitAppended(partition, address, deps) {
|
|
3317
|
-
await partition.appendChain;
|
|
3318
|
-
const failed = partition.failed ?? deps.fatalError();
|
|
3319
|
-
if (failed !== null) throw failed;
|
|
3320
|
-
try {
|
|
3321
|
-
await partition.walWriter.commit();
|
|
3322
|
-
} catch (err) {
|
|
3323
|
-
throw failPartition(partition, address, deps, err);
|
|
3324
|
-
}
|
|
3201
|
+
};
|
|
3325
3202
|
}
|
|
3326
3203
|
|
|
3327
3204
|
// src/persistence/durability/mutation-entry.ts
|
|
@@ -4689,7 +4566,7 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
|
|
|
4689
4566
|
// src/persistence/durability/manager.ts
|
|
4690
4567
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
4691
4568
|
const sizes = createStoredSizesRecord(true);
|
|
4692
|
-
const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
|
|
4569
|
+
const logBytes = createLogBytesLedger(directoryOverride ?? createDurableDirectory(config.directory ?? ""), sizes);
|
|
4693
4570
|
const { directory } = logBytes;
|
|
4694
4571
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
4695
4572
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
@@ -4813,10 +4690,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4813
4690
|
defersCommit: (record) => commitDeferral?.(record) === true
|
|
4814
4691
|
};
|
|
4815
4692
|
async function recordMutations(records) {
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
return records.map(() => ({ ok: false, error }));
|
|
4819
|
-
}
|
|
4693
|
+
const refusal = fatalError ?? (shuttingDown ? shutDownRefusal() : null);
|
|
4694
|
+
if (refusal !== null) return records.map(() => ({ ok: false, error: refusal }));
|
|
4820
4695
|
const outcomes = await recordBatchesByPartition(
|
|
4821
4696
|
records,
|
|
4822
4697
|
(record) => getOrCreatePartition(record.indexName, record.partitionId, 0),
|
|
@@ -4931,7 +4806,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4931
4806
|
return;
|
|
4932
4807
|
}
|
|
4933
4808
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
4934
|
-
await directory
|
|
4809
|
+
await writeIndexMetadataFile(directory, indexName, bytes);
|
|
4935
4810
|
getOrCreateIndexState(indexName);
|
|
4936
4811
|
});
|
|
4937
4812
|
},
|
|
@@ -4968,14 +4843,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4968
4843
|
async shutdown() {
|
|
4969
4844
|
shuttingDown = true;
|
|
4970
4845
|
stopTimers();
|
|
4846
|
+
for (const indexState of indexes.values()) indexState.unloading = true;
|
|
4971
4847
|
for (const indexState of indexes.values()) {
|
|
4972
4848
|
if (indexState.checkpointInFlight !== null) {
|
|
4973
4849
|
await indexState.checkpointInFlight.catch(() => void 0);
|
|
4974
4850
|
}
|
|
4975
4851
|
for (const partition of indexState.partitions.values()) {
|
|
4852
|
+
await partition.appendChain;
|
|
4976
4853
|
await closeWriterReportingFailure(closePartitionWriter(partition));
|
|
4977
4854
|
}
|
|
4978
4855
|
}
|
|
4856
|
+
await metadataWrites.settleAll();
|
|
4979
4857
|
indexes.clear();
|
|
4980
4858
|
terminateCheckpointWorker();
|
|
4981
4859
|
await directoryWatch.release();
|
|
@@ -4983,6 +4861,1061 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4983
4861
|
};
|
|
4984
4862
|
}
|
|
4985
4863
|
|
|
4864
|
+
// src/persistence/durability/indexeddb-directory/connection.ts
|
|
4865
|
+
function requestResult(request) {
|
|
4866
|
+
return new Promise((resolve, reject) => {
|
|
4867
|
+
request.onsuccess = () => resolve(request.result);
|
|
4868
|
+
request.onerror = () => reject(request.error);
|
|
4869
|
+
});
|
|
4870
|
+
}
|
|
4871
|
+
function causeOf(error) {
|
|
4872
|
+
if (error instanceof Error || typeof error === "object" && error !== null && "message" in error) {
|
|
4873
|
+
return String(error.message);
|
|
4874
|
+
}
|
|
4875
|
+
return String(error);
|
|
4876
|
+
}
|
|
4877
|
+
function rootOf(location) {
|
|
4878
|
+
return `indexeddb:${location.databaseName}/${location.storeName}`;
|
|
4879
|
+
}
|
|
4880
|
+
function openLogConnection(location) {
|
|
4881
|
+
const root = rootOf(location);
|
|
4882
|
+
let opening = null;
|
|
4883
|
+
let lostReason = null;
|
|
4884
|
+
let closedByEngine = false;
|
|
4885
|
+
const token = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
4886
|
+
function storeFailed(failure, cause) {
|
|
4887
|
+
return new NarsilError(failure.code, `IndexedDB could not ${failure.action} in ${root}: ${cause}`, {
|
|
4888
|
+
directory: root,
|
|
4889
|
+
cause
|
|
4890
|
+
});
|
|
4891
|
+
}
|
|
4892
|
+
function replaced() {
|
|
4893
|
+
return new NarsilError(
|
|
4894
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
4895
|
+
`The engine treats ${root} as replaced, because ${lostReason ?? "the engine closed it"}, so it acknowledges no later write`,
|
|
4896
|
+
{ directory: root }
|
|
4897
|
+
);
|
|
4898
|
+
}
|
|
4899
|
+
function watch(database2) {
|
|
4900
|
+
database2.onclose = () => {
|
|
4901
|
+
lostReason ??= "the browser closed the connection";
|
|
4902
|
+
};
|
|
4903
|
+
database2.onversionchange = () => {
|
|
4904
|
+
lostReason ??= "another connection asked for a version change of the database";
|
|
4905
|
+
database2.close();
|
|
4906
|
+
};
|
|
4907
|
+
}
|
|
4908
|
+
function open() {
|
|
4909
|
+
const factory = globalThis.indexedDB;
|
|
4910
|
+
if (factory === void 0) {
|
|
4911
|
+
return Promise.reject(
|
|
4912
|
+
new NarsilError(ErrorCodes.CONFIG_INVALID, "IndexedDB is not available, so the write-ahead log cannot open")
|
|
4913
|
+
);
|
|
4914
|
+
}
|
|
4915
|
+
return new Promise((resolve, reject) => {
|
|
4916
|
+
const request = factory.open(location.databaseName);
|
|
4917
|
+
let blockedTimer = null;
|
|
4918
|
+
const settle = () => {
|
|
4919
|
+
if (blockedTimer !== null) clearTimeout(blockedTimer);
|
|
4920
|
+
};
|
|
4921
|
+
request.onblocked = () => {
|
|
4922
|
+
blockedTimer = setTimeout(
|
|
4923
|
+
() => reject(storeFailed({ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" }, "the open is blocked")),
|
|
4924
|
+
INDEXEDDB_BLOCKED_TIMEOUT_MS
|
|
4925
|
+
);
|
|
4926
|
+
};
|
|
4927
|
+
request.onupgradeneeded = () => {
|
|
4928
|
+
const database2 = request.result;
|
|
4929
|
+
if (!database2.objectStoreNames.contains(location.storeName)) database2.createObjectStore(location.storeName);
|
|
4930
|
+
};
|
|
4931
|
+
request.onsuccess = () => {
|
|
4932
|
+
settle();
|
|
4933
|
+
const database2 = request.result;
|
|
4934
|
+
if (!database2.objectStoreNames.contains(location.storeName)) {
|
|
4935
|
+
database2.close();
|
|
4936
|
+
reject(
|
|
4937
|
+
storeFailed(
|
|
4938
|
+
{ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" },
|
|
4939
|
+
`the database holds no object store named "${location.storeName}"`
|
|
4940
|
+
)
|
|
4941
|
+
);
|
|
4942
|
+
return;
|
|
4943
|
+
}
|
|
4944
|
+
watch(database2);
|
|
4945
|
+
resolve(database2);
|
|
4946
|
+
};
|
|
4947
|
+
request.onerror = () => {
|
|
4948
|
+
settle();
|
|
4949
|
+
reject(storeFailed({ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open" }, causeOf(request.error)));
|
|
4950
|
+
};
|
|
4951
|
+
});
|
|
4952
|
+
}
|
|
4953
|
+
async function database() {
|
|
4954
|
+
if (closedByEngine || lostReason !== null) throw replaced();
|
|
4955
|
+
if (opening === null) {
|
|
4956
|
+
opening = open();
|
|
4957
|
+
opening.catch(() => {
|
|
4958
|
+
opening = null;
|
|
4959
|
+
});
|
|
4960
|
+
}
|
|
4961
|
+
const opened = await opening;
|
|
4962
|
+
if (lostReason !== null || closedByEngine) throw replaced();
|
|
4963
|
+
return opened;
|
|
4964
|
+
}
|
|
4965
|
+
async function transact(mode, durability, failure, body) {
|
|
4966
|
+
const opened = await database();
|
|
4967
|
+
let transaction;
|
|
4968
|
+
try {
|
|
4969
|
+
transaction = durability === null ? opened.transaction(location.storeName, mode) : opened.transaction(location.storeName, mode, { durability });
|
|
4970
|
+
} catch (error) {
|
|
4971
|
+
if (lostReason === null && error.name === "InvalidStateError") {
|
|
4972
|
+
lostReason = "the browser closed the connection";
|
|
4973
|
+
}
|
|
4974
|
+
throw lostReason === null ? storeFailed(failure, causeOf(error)) : replaced();
|
|
4975
|
+
}
|
|
4976
|
+
const committed = new Promise((resolve, reject) => {
|
|
4977
|
+
transaction.oncomplete = () => resolve();
|
|
4978
|
+
transaction.onabort = () => reject(transaction.error ?? new Error("the transaction aborted"));
|
|
4979
|
+
});
|
|
4980
|
+
let result;
|
|
4981
|
+
try {
|
|
4982
|
+
result = await body(transaction.objectStore(location.storeName));
|
|
4983
|
+
} catch (error) {
|
|
4984
|
+
try {
|
|
4985
|
+
transaction.abort();
|
|
4986
|
+
} catch {
|
|
4987
|
+
}
|
|
4988
|
+
await committed.catch(() => void 0);
|
|
4989
|
+
throw transactionFailed(transaction, failure, error);
|
|
4990
|
+
}
|
|
4991
|
+
try {
|
|
4992
|
+
await committed;
|
|
4993
|
+
} catch (error) {
|
|
4994
|
+
throw transactionFailed(transaction, failure, error);
|
|
4995
|
+
}
|
|
4996
|
+
if (lostReason !== null || closedByEngine) throw replaced();
|
|
4997
|
+
return result;
|
|
4998
|
+
}
|
|
4999
|
+
function transactionFailed(transaction, failure, error) {
|
|
5000
|
+
const aborted = transaction.error?.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
5001
|
+
if (lostReason === null && aborted) {
|
|
5002
|
+
lostReason = "the browser closed the connection";
|
|
5003
|
+
}
|
|
5004
|
+
if (lostReason !== null || closedByEngine) return replaced();
|
|
5005
|
+
return error instanceof NarsilError ? error : storeFailed(failure, causeOf(error));
|
|
5006
|
+
}
|
|
5007
|
+
return {
|
|
5008
|
+
root,
|
|
5009
|
+
read: (failure, body) => transact("readonly", null, failure, body),
|
|
5010
|
+
write: (durability, failure, body) => transact("readwrite", durability, failure, body),
|
|
5011
|
+
async identity() {
|
|
5012
|
+
await database().catch((error) => {
|
|
5013
|
+
if (lostReason === null && !closedByEngine) throw error;
|
|
5014
|
+
});
|
|
5015
|
+
return lostReason === null && !closedByEngine ? token : null;
|
|
5016
|
+
},
|
|
5017
|
+
close() {
|
|
5018
|
+
closedByEngine = true;
|
|
5019
|
+
const pending = opening;
|
|
5020
|
+
opening = null;
|
|
5021
|
+
void pending?.then(
|
|
5022
|
+
(opened) => opened.close(),
|
|
5023
|
+
() => void 0
|
|
5024
|
+
);
|
|
5025
|
+
}
|
|
5026
|
+
};
|
|
5027
|
+
}
|
|
5028
|
+
|
|
5029
|
+
// src/persistence/durability/indexeddb-directory/records.ts
|
|
5030
|
+
function appendKey(key, n) {
|
|
5031
|
+
return [key, n];
|
|
5032
|
+
}
|
|
5033
|
+
function appendRecordsOf(key) {
|
|
5034
|
+
return IDBKeyRange.bound([key], [key, []], true, true);
|
|
5035
|
+
}
|
|
5036
|
+
function compactBytes(bytes) {
|
|
5037
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? bytes : bytes.slice();
|
|
5038
|
+
}
|
|
5039
|
+
function joinBytes(chunks) {
|
|
5040
|
+
if (chunks.length === 1) return compactBytes(chunks[0]);
|
|
5041
|
+
let total = 0;
|
|
5042
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
5043
|
+
const joined = new Uint8Array(total);
|
|
5044
|
+
let offset = 0;
|
|
5045
|
+
for (const chunk of chunks) {
|
|
5046
|
+
joined.set(chunk, offset);
|
|
5047
|
+
offset += chunk.byteLength;
|
|
5048
|
+
}
|
|
5049
|
+
return joined;
|
|
5050
|
+
}
|
|
5051
|
+
function requireBytes(key, value) {
|
|
5052
|
+
if (value instanceof Uint8Array) return value;
|
|
5053
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The IndexedDB record of "${key}" holds no bytes`, { key });
|
|
5054
|
+
}
|
|
5055
|
+
function requireAppendNumber(key, recordKey) {
|
|
5056
|
+
const n = Array.isArray(recordKey) && recordKey.length === 2 && recordKey[0] === key ? recordKey[1] : void 0;
|
|
5057
|
+
if (typeof n === "number" && Number.isSafeInteger(n) && n >= 1) return n;
|
|
5058
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `An append record of "${key}" carries no valid position`, {
|
|
5059
|
+
key
|
|
5060
|
+
});
|
|
5061
|
+
}
|
|
5062
|
+
async function readRecords(store, key) {
|
|
5063
|
+
const first = await requestResult(store.get(key));
|
|
5064
|
+
if (first === void 0 && await requestResult(store.getKey(key)) === void 0) return null;
|
|
5065
|
+
const bytes = requireBytes(key, first);
|
|
5066
|
+
const range = appendRecordsOf(key);
|
|
5067
|
+
const keys = await requestResult(store.getAllKeys(range));
|
|
5068
|
+
const values = await requestResult(store.getAll(range));
|
|
5069
|
+
if (keys.length !== values.length) {
|
|
5070
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The append records of "${key}" changed during a read`, {
|
|
5071
|
+
key
|
|
5072
|
+
});
|
|
5073
|
+
}
|
|
5074
|
+
const appended = [];
|
|
5075
|
+
for (let at = 0; at < keys.length; at++) {
|
|
5076
|
+
appended.push({ n: requireAppendNumber(key, keys[at]), bytes: requireBytes(key, values[at]) });
|
|
5077
|
+
}
|
|
5078
|
+
return { first: bytes, appended };
|
|
5079
|
+
}
|
|
5080
|
+
function lengthsOf(records) {
|
|
5081
|
+
return [
|
|
5082
|
+
{ n: 0, length: records.first.byteLength },
|
|
5083
|
+
...records.appended.map((record) => ({ n: record.n, length: record.bytes.byteLength }))
|
|
5084
|
+
];
|
|
5085
|
+
}
|
|
5086
|
+
function walkCursor(store, key, range, visit) {
|
|
5087
|
+
return new Promise((resolve, reject) => {
|
|
5088
|
+
const request = store.openCursor(range);
|
|
5089
|
+
request.onsuccess = () => {
|
|
5090
|
+
const cursor = request.result;
|
|
5091
|
+
if (cursor === null) {
|
|
5092
|
+
resolve(true);
|
|
5093
|
+
return;
|
|
5094
|
+
}
|
|
5095
|
+
let enough;
|
|
5096
|
+
try {
|
|
5097
|
+
enough = visit(requireAppendNumber(key, cursor.key), requireBytes(key, cursor.value));
|
|
5098
|
+
} catch (error) {
|
|
5099
|
+
reject(error);
|
|
5100
|
+
return;
|
|
5101
|
+
}
|
|
5102
|
+
if (enough) resolve(false);
|
|
5103
|
+
else cursor.continue();
|
|
5104
|
+
};
|
|
5105
|
+
request.onerror = () => reject(request.error);
|
|
5106
|
+
});
|
|
5107
|
+
}
|
|
5108
|
+
async function walkBlocks(store, key, afterN, visit) {
|
|
5109
|
+
if (afterN === null) {
|
|
5110
|
+
const first = await requestResult(store.get(key));
|
|
5111
|
+
if (first === void 0 && await requestResult(store.getKey(key)) === void 0) return null;
|
|
5112
|
+
if (visit(0, requireBytes(key, first))) return false;
|
|
5113
|
+
return walkCursor(store, key, appendRecordsOf(key), visit);
|
|
5114
|
+
}
|
|
5115
|
+
return walkCursor(store, key, IDBKeyRange.bound(appendKey(key, afterN), [key, []], true, true), visit);
|
|
5116
|
+
}
|
|
5117
|
+
function blocksChanged(key) {
|
|
5118
|
+
return new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The blocks of "${key}" changed during a read`, { key });
|
|
5119
|
+
}
|
|
5120
|
+
async function readBlocks(store, key, wanted) {
|
|
5121
|
+
const found = /* @__PURE__ */ new Map();
|
|
5122
|
+
const appended = wanted.filter((block) => block.n > 0);
|
|
5123
|
+
const reads = [];
|
|
5124
|
+
if (appended.length < wanted.length) {
|
|
5125
|
+
reads.push(
|
|
5126
|
+
requestResult(store.get(key)).then((value) => {
|
|
5127
|
+
found.set(0, requireBytes(key, value));
|
|
5128
|
+
})
|
|
5129
|
+
);
|
|
5130
|
+
}
|
|
5131
|
+
const lowest = appended[0];
|
|
5132
|
+
const highest = appended[appended.length - 1];
|
|
5133
|
+
if (lowest !== void 0 && highest !== void 0) {
|
|
5134
|
+
const range = IDBKeyRange.bound(appendKey(key, lowest.n), appendKey(key, highest.n));
|
|
5135
|
+
reads.push(
|
|
5136
|
+
Promise.all([requestResult(store.getAllKeys(range)), requestResult(store.getAll(range))]).then(
|
|
5137
|
+
([keys, values]) => {
|
|
5138
|
+
if (keys.length !== values.length) throw blocksChanged(key);
|
|
5139
|
+
for (let at = 0; at < keys.length; at++) {
|
|
5140
|
+
found.set(requireAppendNumber(key, keys[at]), requireBytes(key, values[at]));
|
|
5141
|
+
}
|
|
5142
|
+
}
|
|
5143
|
+
)
|
|
5144
|
+
);
|
|
5145
|
+
}
|
|
5146
|
+
await Promise.all(reads);
|
|
5147
|
+
for (const block of wanted) {
|
|
5148
|
+
if (found.get(block.n)?.byteLength !== block.length) throw blocksChanged(key);
|
|
5149
|
+
}
|
|
5150
|
+
return found;
|
|
5151
|
+
}
|
|
5152
|
+
function splitIntoBlocks(chunks, blockBytes) {
|
|
5153
|
+
let total = 0;
|
|
5154
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
5155
|
+
const blocks = [];
|
|
5156
|
+
let block = new Uint8Array(Math.min(blockBytes, total));
|
|
5157
|
+
let filled = 0;
|
|
5158
|
+
for (const chunk of chunks) {
|
|
5159
|
+
let read = 0;
|
|
5160
|
+
while (read < chunk.byteLength) {
|
|
5161
|
+
const taken = Math.min(block.byteLength - filled, chunk.byteLength - read);
|
|
5162
|
+
block.set(chunk.subarray(read, read + taken), filled);
|
|
5163
|
+
filled += taken;
|
|
5164
|
+
read += taken;
|
|
5165
|
+
total -= taken;
|
|
5166
|
+
if (filled === block.byteLength && total > 0) {
|
|
5167
|
+
blocks.push(block);
|
|
5168
|
+
block = new Uint8Array(Math.min(blockBytes, total));
|
|
5169
|
+
filled = 0;
|
|
5170
|
+
}
|
|
5171
|
+
}
|
|
5172
|
+
}
|
|
5173
|
+
blocks.push(block);
|
|
5174
|
+
return blocks;
|
|
5175
|
+
}
|
|
5176
|
+
function contentOf(records) {
|
|
5177
|
+
if (records.appended.length === 0) return records.first;
|
|
5178
|
+
return joinBytes([records.first, ...records.appended.map((record) => record.bytes)]);
|
|
5179
|
+
}
|
|
5180
|
+
async function removeRecords(store, key) {
|
|
5181
|
+
await requestResult(store.delete(appendRecordsOf(key)));
|
|
5182
|
+
await requestResult(store.delete(key));
|
|
5183
|
+
}
|
|
5184
|
+
async function replaceRecords(store, key, records) {
|
|
5185
|
+
await removeRecords(store, key);
|
|
5186
|
+
await requestResult(store.put(records.first, key));
|
|
5187
|
+
for (const record of records.appended) await requestResult(store.put(record.bytes, appendKey(key, record.n)));
|
|
5188
|
+
}
|
|
5189
|
+
function listKeys(store, prefix) {
|
|
5190
|
+
return new Promise((resolve, reject) => {
|
|
5191
|
+
const keys = [];
|
|
5192
|
+
const request = store.openKeyCursor(IDBKeyRange.lowerBound(prefix));
|
|
5193
|
+
request.onsuccess = () => {
|
|
5194
|
+
const cursor = request.result;
|
|
5195
|
+
if (cursor === null || typeof cursor.key !== "string" || !cursor.key.startsWith(prefix)) {
|
|
5196
|
+
resolve(keys);
|
|
5197
|
+
return;
|
|
5198
|
+
}
|
|
5199
|
+
keys.push(cursor.key);
|
|
5200
|
+
cursor.continue();
|
|
5201
|
+
};
|
|
5202
|
+
request.onerror = () => reject(request.error);
|
|
5203
|
+
});
|
|
5204
|
+
}
|
|
5205
|
+
|
|
5206
|
+
// src/persistence/durability/indexeddb-directory/append-handle.ts
|
|
5207
|
+
async function openFileRecords(connection, key) {
|
|
5208
|
+
const parts = await connection.write(
|
|
5209
|
+
"strict",
|
|
5210
|
+
{ code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "open a log file" },
|
|
5211
|
+
async (store) => {
|
|
5212
|
+
const records = await readRecords(store, key);
|
|
5213
|
+
if (records !== null) return lengthsOf(records);
|
|
5214
|
+
await requestResult(store.put(new Uint8Array(0), key));
|
|
5215
|
+
return [{ n: 0, length: 0 }];
|
|
5216
|
+
}
|
|
5217
|
+
);
|
|
5218
|
+
const nextN = parts[parts.length - 1].n + 1;
|
|
5219
|
+
return { key, parts, unsynced: [], nextN, handles: 0, detached: false, failure: null, moving: null };
|
|
5220
|
+
}
|
|
5221
|
+
async function waitForMove(file) {
|
|
5222
|
+
while (file.moving !== null) await file.moving;
|
|
5223
|
+
}
|
|
5224
|
+
function lengthsOfRecords(records) {
|
|
5225
|
+
return records.map((record) => ({ n: record.n, length: record.bytes.byteLength }));
|
|
5226
|
+
}
|
|
5227
|
+
function sizeOf(file) {
|
|
5228
|
+
let size = 0;
|
|
5229
|
+
for (const part of file.parts) size += part.length;
|
|
5230
|
+
return size;
|
|
5231
|
+
}
|
|
5232
|
+
function recordKeyOf(key, n) {
|
|
5233
|
+
return n === 0 ? key : appendKey(key, n);
|
|
5234
|
+
}
|
|
5235
|
+
function putRecords(store, key, records) {
|
|
5236
|
+
return Promise.all(records.map((record) => requestResult(store.put(record.bytes, recordKeyOf(key, record.n)))));
|
|
5237
|
+
}
|
|
5238
|
+
function holdsNoBytes(file) {
|
|
5239
|
+
return file.parts.length === 1 && file.parts[0].length === 0;
|
|
5240
|
+
}
|
|
5241
|
+
function createAppendHandle(file, deps) {
|
|
5242
|
+
const { connection, blocks } = deps;
|
|
5243
|
+
const appendFailure = { code: ErrorCodes.PERSISTENCE_SAVE_FAILED, action: "append to a log file" };
|
|
5244
|
+
const strictCommitFailure = {
|
|
5245
|
+
code: ErrorCodes.PERSISTENCE_FSYNC_FAILED,
|
|
5246
|
+
action: "commit a log file with strict durability"
|
|
5247
|
+
};
|
|
5248
|
+
let closed = false;
|
|
5249
|
+
file.handles += 1;
|
|
5250
|
+
function requireUsable() {
|
|
5251
|
+
if (closed) throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The log file "${file.key}" is closed`);
|
|
5252
|
+
if (file.failure !== null) throw file.failure;
|
|
5253
|
+
if (file.detached) {
|
|
5254
|
+
throw new NarsilError(
|
|
5255
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
5256
|
+
`The log file "${file.key}" was removed or replaced while a handle held it open`,
|
|
5257
|
+
{ key: file.key }
|
|
5258
|
+
);
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
function latch(error) {
|
|
5262
|
+
const failed = error instanceof Error ? error : new Error(String(error));
|
|
5263
|
+
file.failure ??= failed;
|
|
5264
|
+
return file.failure;
|
|
5265
|
+
}
|
|
5266
|
+
async function putCommitted(durability, failure, records) {
|
|
5267
|
+
const key = file.key;
|
|
5268
|
+
const epoch = blocks.appendEpochOf(key);
|
|
5269
|
+
try {
|
|
5270
|
+
await connection.write(durability, failure, (store) => putRecords(store, key, records));
|
|
5271
|
+
} catch (error) {
|
|
5272
|
+
throw latch(error);
|
|
5273
|
+
}
|
|
5274
|
+
blocks.appended(key, epoch, lengthsOfRecords(records));
|
|
5275
|
+
}
|
|
5276
|
+
async function writeUnsynced(durability) {
|
|
5277
|
+
if (file.moving !== null) await waitForMove(file);
|
|
5278
|
+
const batch = file.unsynced.slice();
|
|
5279
|
+
if (batch.length === 0) return;
|
|
5280
|
+
await putCommitted(durability, durability === "strict" ? strictCommitFailure : appendFailure, batch);
|
|
5281
|
+
file.unsynced.splice(0, batch.length);
|
|
5282
|
+
}
|
|
5283
|
+
async function truncateTo(byteLength) {
|
|
5284
|
+
if (file.moving !== null) await waitForMove(file);
|
|
5285
|
+
const key = file.key;
|
|
5286
|
+
const batch = file.unsynced.slice();
|
|
5287
|
+
const kept = [];
|
|
5288
|
+
let start = 0;
|
|
5289
|
+
for (const part of file.parts) {
|
|
5290
|
+
if (start >= byteLength && part.n !== 0) break;
|
|
5291
|
+
kept.push({ ...part, length: Math.min(part.length, byteLength - start) });
|
|
5292
|
+
start += part.length;
|
|
5293
|
+
}
|
|
5294
|
+
const dropped = file.parts.slice(kept.length);
|
|
5295
|
+
const cut = kept[kept.length - 1];
|
|
5296
|
+
const cutLength = cut.length;
|
|
5297
|
+
const cutIsPartial = cutLength < (file.parts[kept.length - 1]?.length ?? 0);
|
|
5298
|
+
const epoch = blocks.begin(key);
|
|
5299
|
+
try {
|
|
5300
|
+
await connection.write("strict", strictCommitFailure, async (store) => {
|
|
5301
|
+
await putRecords(store, key, batch);
|
|
5302
|
+
for (const part of dropped) await requestResult(store.delete(recordKeyOf(key, part.n)));
|
|
5303
|
+
if (!cutIsPartial) return;
|
|
5304
|
+
const recordKey = recordKeyOf(key, cut.n);
|
|
5305
|
+
const stored = await requestResult(store.get(recordKey));
|
|
5306
|
+
if (!(stored instanceof Uint8Array) || stored.byteLength < cutLength) {
|
|
5307
|
+
throw new NarsilError(
|
|
5308
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5309
|
+
`A record of "${key}" holds fewer bytes than its log`,
|
|
5310
|
+
{
|
|
5311
|
+
key
|
|
5312
|
+
}
|
|
5313
|
+
);
|
|
5314
|
+
}
|
|
5315
|
+
await requestResult(store.put(stored.slice(0, cutLength), recordKey));
|
|
5316
|
+
});
|
|
5317
|
+
} catch (error) {
|
|
5318
|
+
blocks.settle(key, epoch, void 0);
|
|
5319
|
+
throw error;
|
|
5320
|
+
}
|
|
5321
|
+
blocks.settle(key, epoch, kept);
|
|
5322
|
+
file.parts = kept;
|
|
5323
|
+
file.unsynced = [];
|
|
5324
|
+
file.nextN = kept[kept.length - 1].n + 1;
|
|
5325
|
+
}
|
|
5326
|
+
return {
|
|
5327
|
+
async append(bytes) {
|
|
5328
|
+
requireUsable();
|
|
5329
|
+
if (bytes.byteLength === 0) return;
|
|
5330
|
+
const records = [];
|
|
5331
|
+
for (const piece of splitIntoBlocks([bytes], INDEXEDDB_BLOCK_BYTES)) {
|
|
5332
|
+
if (holdsNoBytes(file)) {
|
|
5333
|
+
file.parts[0] = { n: 0, length: piece.byteLength };
|
|
5334
|
+
records.push({ n: 0, bytes: piece });
|
|
5335
|
+
continue;
|
|
5336
|
+
}
|
|
5337
|
+
records.push({ n: file.nextN, bytes: piece });
|
|
5338
|
+
file.parts.push({ n: file.nextN, length: piece.byteLength });
|
|
5339
|
+
file.nextN += 1;
|
|
5340
|
+
}
|
|
5341
|
+
file.unsynced.push(...records);
|
|
5342
|
+
if (!deps.relaxedAppends) return;
|
|
5343
|
+
if (file.moving !== null) await waitForMove(file);
|
|
5344
|
+
await putCommitted("relaxed", appendFailure, records);
|
|
5345
|
+
},
|
|
5346
|
+
async sync() {
|
|
5347
|
+
requireUsable();
|
|
5348
|
+
await writeUnsynced("strict");
|
|
5349
|
+
},
|
|
5350
|
+
async truncate(byteLength) {
|
|
5351
|
+
requireUsable();
|
|
5352
|
+
if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > sizeOf(file)) {
|
|
5353
|
+
throw new NarsilError(
|
|
5354
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
5355
|
+
`The log file "${file.key}" cannot be cut to ${byteLength} bytes, because it holds ${sizeOf(file)}`,
|
|
5356
|
+
{ key: file.key, byteLength }
|
|
5357
|
+
);
|
|
5358
|
+
}
|
|
5359
|
+
try {
|
|
5360
|
+
await truncateTo(byteLength);
|
|
5361
|
+
} catch (error) {
|
|
5362
|
+
throw latch(error);
|
|
5363
|
+
}
|
|
5364
|
+
},
|
|
5365
|
+
async size() {
|
|
5366
|
+
if (closed) throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The log file "${file.key}" is closed`);
|
|
5367
|
+
return sizeOf(file);
|
|
5368
|
+
},
|
|
5369
|
+
async close() {
|
|
5370
|
+
if (closed) return;
|
|
5371
|
+
closed = true;
|
|
5372
|
+
file.handles -= 1;
|
|
5373
|
+
try {
|
|
5374
|
+
if (!deps.relaxedAppends && file.failure === null && !file.detached) await writeUnsynced("relaxed");
|
|
5375
|
+
} finally {
|
|
5376
|
+
if (file.handles === 0) deps.release(file);
|
|
5377
|
+
}
|
|
5378
|
+
}
|
|
5379
|
+
};
|
|
5380
|
+
}
|
|
5381
|
+
|
|
5382
|
+
// src/persistence/durability/indexeddb-directory/block-map.ts
|
|
5383
|
+
function laidOut(lengths) {
|
|
5384
|
+
const blocks = [];
|
|
5385
|
+
let start = 0;
|
|
5386
|
+
for (const { n, length } of lengths) {
|
|
5387
|
+
blocks.push({ n, length, start });
|
|
5388
|
+
start += length;
|
|
5389
|
+
}
|
|
5390
|
+
return blocks;
|
|
5391
|
+
}
|
|
5392
|
+
function sizeOfBlocks(blocks) {
|
|
5393
|
+
const last = blocks[blocks.length - 1];
|
|
5394
|
+
return last === void 0 ? 0 : last.start + last.length;
|
|
5395
|
+
}
|
|
5396
|
+
function firstBlockAt(blocks, offset) {
|
|
5397
|
+
let low = 0;
|
|
5398
|
+
let high = blocks.length - 1;
|
|
5399
|
+
while (low < high) {
|
|
5400
|
+
const middle = low + high + 1 >>> 1;
|
|
5401
|
+
if (blocks[middle].start <= offset) low = middle;
|
|
5402
|
+
else high = middle - 1;
|
|
5403
|
+
}
|
|
5404
|
+
return low;
|
|
5405
|
+
}
|
|
5406
|
+
function blockNumbered(blocks, n) {
|
|
5407
|
+
let low = 0;
|
|
5408
|
+
let high = blocks.length - 1;
|
|
5409
|
+
while (low <= high) {
|
|
5410
|
+
const middle = low + high >>> 1;
|
|
5411
|
+
const block = blocks[middle];
|
|
5412
|
+
if (block.n === n) return block;
|
|
5413
|
+
if (block.n < n) low = middle + 1;
|
|
5414
|
+
else high = middle - 1;
|
|
5415
|
+
}
|
|
5416
|
+
return void 0;
|
|
5417
|
+
}
|
|
5418
|
+
function mergedBlocks(known, complete, lengths) {
|
|
5419
|
+
let merged = known;
|
|
5420
|
+
for (const { n, length } of lengths) {
|
|
5421
|
+
const last = merged[merged.length - 1];
|
|
5422
|
+
if (last === void 0) return void 0;
|
|
5423
|
+
if (n > last.n) {
|
|
5424
|
+
if (complete) merged.push({ n, length, start: last.start + last.length });
|
|
5425
|
+
continue;
|
|
5426
|
+
}
|
|
5427
|
+
if (n === 0 && merged.length === 1 && last.length === 0) {
|
|
5428
|
+
if (!complete) return void 0;
|
|
5429
|
+
merged = [{ n: 0, length, start: 0 }];
|
|
5430
|
+
continue;
|
|
5431
|
+
}
|
|
5432
|
+
const existing = blockNumbered(merged, n);
|
|
5433
|
+
if (existing === void 0 || existing.length !== length) return void 0;
|
|
5434
|
+
}
|
|
5435
|
+
return merged;
|
|
5436
|
+
}
|
|
5437
|
+
function createBlockMap() {
|
|
5438
|
+
const states = /* @__PURE__ */ new Map();
|
|
5439
|
+
const cache = /* @__PURE__ */ new Map();
|
|
5440
|
+
let counter = 0;
|
|
5441
|
+
function next() {
|
|
5442
|
+
counter += 1;
|
|
5443
|
+
return counter;
|
|
5444
|
+
}
|
|
5445
|
+
function stateOf(key) {
|
|
5446
|
+
const known = states.get(key);
|
|
5447
|
+
if (known !== void 0) return known;
|
|
5448
|
+
const created = { generation: next(), epoch: next(), changes: 0, blocks: void 0, complete: false };
|
|
5449
|
+
states.set(key, created);
|
|
5450
|
+
return created;
|
|
5451
|
+
}
|
|
5452
|
+
function cacheKeyOf(key, n) {
|
|
5453
|
+
return `${n}\0${key}`;
|
|
5454
|
+
}
|
|
5455
|
+
function invalidate(state) {
|
|
5456
|
+
state.generation = next();
|
|
5457
|
+
state.changes += 1;
|
|
5458
|
+
state.blocks = void 0;
|
|
5459
|
+
state.complete = false;
|
|
5460
|
+
}
|
|
5461
|
+
return {
|
|
5462
|
+
viewOf(key) {
|
|
5463
|
+
const state = stateOf(key);
|
|
5464
|
+
const layout = state.blocks === void 0 ? void 0 : { blocks: state.blocks, complete: state.complete };
|
|
5465
|
+
return { generation: state.generation, changes: state.changes, layout };
|
|
5466
|
+
},
|
|
5467
|
+
generationOf: (key) => stateOf(key).generation,
|
|
5468
|
+
install(key, view, blocks, reachedEnd) {
|
|
5469
|
+
const state = stateOf(key);
|
|
5470
|
+
if (state.generation !== view.generation) return;
|
|
5471
|
+
const complete = reachedEnd && state.changes === view.changes;
|
|
5472
|
+
const current = state.blocks;
|
|
5473
|
+
if (current !== void 0 && (state.complete || !complete && blocks.length <= current.length)) return;
|
|
5474
|
+
state.blocks = blocks;
|
|
5475
|
+
state.complete = complete;
|
|
5476
|
+
},
|
|
5477
|
+
forget: (key) => invalidate(stateOf(key)),
|
|
5478
|
+
begin(key) {
|
|
5479
|
+
const state = stateOf(key);
|
|
5480
|
+
invalidate(state);
|
|
5481
|
+
state.epoch = next();
|
|
5482
|
+
return state.epoch;
|
|
5483
|
+
},
|
|
5484
|
+
settle(key, epoch, lengths) {
|
|
5485
|
+
const state = stateOf(key);
|
|
5486
|
+
if (state.epoch !== epoch) return;
|
|
5487
|
+
invalidate(state);
|
|
5488
|
+
if (lengths === void 0) return;
|
|
5489
|
+
state.blocks = laidOut(lengths);
|
|
5490
|
+
state.complete = true;
|
|
5491
|
+
},
|
|
5492
|
+
appendEpochOf: (key) => stateOf(key).epoch,
|
|
5493
|
+
appended(key, epoch, lengths) {
|
|
5494
|
+
const state = stateOf(key);
|
|
5495
|
+
if (state.epoch !== epoch) return;
|
|
5496
|
+
state.changes += 1;
|
|
5497
|
+
if (state.blocks === void 0) return;
|
|
5498
|
+
const merged = mergedBlocks(state.blocks, state.complete, lengths);
|
|
5499
|
+
if (merged === void 0) invalidate(state);
|
|
5500
|
+
else state.blocks = merged;
|
|
5501
|
+
},
|
|
5502
|
+
covering(key, generation, blocks, offset, length) {
|
|
5503
|
+
const covered = [];
|
|
5504
|
+
const end = offset + length;
|
|
5505
|
+
for (let at = firstBlockAt(blocks, offset); at < blocks.length; at++) {
|
|
5506
|
+
const block = blocks[at];
|
|
5507
|
+
if (block.start >= end && covered.length > 0) break;
|
|
5508
|
+
const cacheKey = cacheKeyOf(key, block.n);
|
|
5509
|
+
const entry = cache.get(cacheKey);
|
|
5510
|
+
const cached = entry !== void 0 && entry.generation === generation && entry.bytes.byteLength === block.length ? entry.bytes : void 0;
|
|
5511
|
+
if (cached !== void 0) {
|
|
5512
|
+
cache.delete(cacheKey);
|
|
5513
|
+
cache.set(cacheKey, { generation, bytes: cached });
|
|
5514
|
+
}
|
|
5515
|
+
covered.push({ ...block, cached });
|
|
5516
|
+
if (block.start + block.length >= end) break;
|
|
5517
|
+
}
|
|
5518
|
+
return covered;
|
|
5519
|
+
},
|
|
5520
|
+
keep(key, generation, n, bytes) {
|
|
5521
|
+
if (bytes.byteLength > INDEXEDDB_BLOCK_BYTES || stateOf(key).generation !== generation) return;
|
|
5522
|
+
const cacheKey = cacheKeyOf(key, n);
|
|
5523
|
+
cache.delete(cacheKey);
|
|
5524
|
+
cache.set(cacheKey, { generation, bytes });
|
|
5525
|
+
while (cache.size > INDEXEDDB_CACHED_BLOCKS) {
|
|
5526
|
+
const oldest = cache.keys().next();
|
|
5527
|
+
if (oldest.done === true) break;
|
|
5528
|
+
cache.delete(oldest.value);
|
|
5529
|
+
}
|
|
5530
|
+
}
|
|
5531
|
+
};
|
|
5532
|
+
}
|
|
5533
|
+
|
|
5534
|
+
// src/persistence/durability/indexeddb-directory/lock.ts
|
|
5535
|
+
var WEB_LOCK_NAME_PREFIX = "narsil-wal";
|
|
5536
|
+
var heldWithoutWebLocks = /* @__PURE__ */ new Set();
|
|
5537
|
+
function webLockNameOf(location) {
|
|
5538
|
+
return `${WEB_LOCK_NAME_PREFIX}${JSON.stringify([location.databaseName, location.storeName])}`;
|
|
5539
|
+
}
|
|
5540
|
+
function heldElsewhere(location) {
|
|
5541
|
+
return new NarsilError(
|
|
5542
|
+
ErrorCodes.CONFIG_INVALID,
|
|
5543
|
+
`Another tab or engine holds the write-ahead log of IndexedDB database "${location.databaseName}" store "${location.storeName}". Several tabs that share one store need an invalidation adapter or durability.tier "snapshot"`,
|
|
5544
|
+
{ databaseName: location.databaseName, storeName: location.storeName }
|
|
5545
|
+
);
|
|
5546
|
+
}
|
|
5547
|
+
function lockWithinProcess(name, location) {
|
|
5548
|
+
if (heldWithoutWebLocks.has(name)) throw heldElsewhere(location);
|
|
5549
|
+
heldWithoutWebLocks.add(name);
|
|
5550
|
+
let held = true;
|
|
5551
|
+
return {
|
|
5552
|
+
holds: async () => held,
|
|
5553
|
+
async release() {
|
|
5554
|
+
if (!held) return;
|
|
5555
|
+
held = false;
|
|
5556
|
+
heldWithoutWebLocks.delete(name);
|
|
5557
|
+
}
|
|
5558
|
+
};
|
|
5559
|
+
}
|
|
5560
|
+
function lockAcrossTabs(locks, name, location) {
|
|
5561
|
+
return new Promise((resolve, reject) => {
|
|
5562
|
+
let held = false;
|
|
5563
|
+
let releaseLock = () => void 0;
|
|
5564
|
+
const releasing = new Promise((settle) => {
|
|
5565
|
+
releaseLock = settle;
|
|
5566
|
+
});
|
|
5567
|
+
locks.request(name, { mode: "exclusive", ifAvailable: true }, (granted) => {
|
|
5568
|
+
if (granted === null) {
|
|
5569
|
+
reject(heldElsewhere(location));
|
|
5570
|
+
return void 0;
|
|
5571
|
+
}
|
|
5572
|
+
held = true;
|
|
5573
|
+
resolve({
|
|
5574
|
+
holds: async () => held,
|
|
5575
|
+
async release() {
|
|
5576
|
+
held = false;
|
|
5577
|
+
releaseLock();
|
|
5578
|
+
}
|
|
5579
|
+
});
|
|
5580
|
+
return releasing;
|
|
5581
|
+
}).then(
|
|
5582
|
+
() => {
|
|
5583
|
+
held = false;
|
|
5584
|
+
},
|
|
5585
|
+
(error) => {
|
|
5586
|
+
held = false;
|
|
5587
|
+
reject(error);
|
|
5588
|
+
}
|
|
5589
|
+
);
|
|
5590
|
+
});
|
|
5591
|
+
}
|
|
5592
|
+
function lockIndexedDBStore(location) {
|
|
5593
|
+
const name = webLockNameOf(location);
|
|
5594
|
+
const locks = globalThis.navigator?.locks;
|
|
5595
|
+
if (locks === void 0) return Promise.resolve().then(() => lockWithinProcess(name, location));
|
|
5596
|
+
return lockAcrossTabs(locks, name, location);
|
|
5597
|
+
}
|
|
5598
|
+
|
|
5599
|
+
// src/persistence/durability/indexeddb-directory/range-read.ts
|
|
5600
|
+
function endsBefore(key, offset, length) {
|
|
5601
|
+
return new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
|
|
5602
|
+
key,
|
|
5603
|
+
offset,
|
|
5604
|
+
length
|
|
5605
|
+
});
|
|
5606
|
+
}
|
|
5607
|
+
function isLoadFailure(error) {
|
|
5608
|
+
return error instanceof NarsilError && error.code === ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
5609
|
+
}
|
|
5610
|
+
function assembled(key, covered, fetched, offset, length) {
|
|
5611
|
+
const bytes = new Uint8Array(length);
|
|
5612
|
+
for (const block of covered) {
|
|
5613
|
+
if (block.length === 0) continue;
|
|
5614
|
+
const source = block.cached ?? fetched.get(block.n);
|
|
5615
|
+
if (source === void 0) throw endsBefore(key, offset, length);
|
|
5616
|
+
const from = Math.max(offset, block.start);
|
|
5617
|
+
const to = Math.min(offset + length, block.start + block.length);
|
|
5618
|
+
bytes.set(source.subarray(from - block.start, to - block.start), from - offset);
|
|
5619
|
+
}
|
|
5620
|
+
return bytes;
|
|
5621
|
+
}
|
|
5622
|
+
function createRangeReads(connection, blocks, readContent) {
|
|
5623
|
+
async function extendTo(key, view, offset, end) {
|
|
5624
|
+
const known = view.layout === void 0 ? [] : view.layout.blocks.slice();
|
|
5625
|
+
const fetched = /* @__PURE__ */ new Map();
|
|
5626
|
+
let size = sizeOfBlocks(known);
|
|
5627
|
+
const last = known[known.length - 1];
|
|
5628
|
+
const reachedEnd = await connection.read(
|
|
5629
|
+
{ code: ErrorCodes.PERSISTENCE_LOAD_FAILED, action: `read the blocks of "${key}"` },
|
|
5630
|
+
(store) => walkBlocks(store, key, last === void 0 ? null : last.n, (n, bytes) => {
|
|
5631
|
+
const start = size;
|
|
5632
|
+
known.push({ n, length: bytes.byteLength, start });
|
|
5633
|
+
size += bytes.byteLength;
|
|
5634
|
+
if (bytes.byteLength > 0 && start < end && size > offset) fetched.set(n, bytes);
|
|
5635
|
+
return size >= end;
|
|
5636
|
+
})
|
|
5637
|
+
);
|
|
5638
|
+
if (reachedEnd === null) return null;
|
|
5639
|
+
blocks.install(key, view, known, reachedEnd);
|
|
5640
|
+
return { blocks: known, fetched };
|
|
5641
|
+
}
|
|
5642
|
+
async function wholeFileRange(key, offset, length) {
|
|
5643
|
+
const content = await readContent(key);
|
|
5644
|
+
if (content === null || offset + length > content.byteLength) throw endsBefore(key, offset, length);
|
|
5645
|
+
return content.slice(offset, offset + length);
|
|
5646
|
+
}
|
|
5647
|
+
async function blockRange(key, offset, length) {
|
|
5648
|
+
const end = offset + length;
|
|
5649
|
+
const view = blocks.viewOf(key);
|
|
5650
|
+
let layout = view.layout?.blocks;
|
|
5651
|
+
let fetched = /* @__PURE__ */ new Map();
|
|
5652
|
+
if (view.layout === void 0 || !view.layout.complete && sizeOfBlocks(view.layout.blocks) < end) {
|
|
5653
|
+
const extended = await extendTo(key, view, offset, end);
|
|
5654
|
+
layout = extended?.blocks;
|
|
5655
|
+
if (extended !== null) fetched = extended.fetched;
|
|
5656
|
+
}
|
|
5657
|
+
if (layout === void 0 || sizeOfBlocks(layout) < end) {
|
|
5658
|
+
if (blocks.generationOf(key) !== view.generation) return wholeFileRange(key, offset, length);
|
|
5659
|
+
throw endsBefore(key, offset, length);
|
|
5660
|
+
}
|
|
5661
|
+
if (length === 0) return new Uint8Array(0);
|
|
5662
|
+
const covered = blocks.covering(key, view.generation, layout, offset, length);
|
|
5663
|
+
const missing2 = covered.filter((block) => block.length > 0 && block.cached === void 0 && !fetched.has(block.n));
|
|
5664
|
+
if (missing2.length > 0) {
|
|
5665
|
+
try {
|
|
5666
|
+
const read = await connection.read(
|
|
5667
|
+
{ code: ErrorCodes.PERSISTENCE_LOAD_FAILED, action: `read "${key}"` },
|
|
5668
|
+
(store) => readBlocks(store, key, missing2)
|
|
5669
|
+
);
|
|
5670
|
+
for (const [n, bytes] of read) fetched.set(n, bytes);
|
|
5671
|
+
} catch (error) {
|
|
5672
|
+
if (!isLoadFailure(error)) throw error;
|
|
5673
|
+
blocks.forget(key);
|
|
5674
|
+
return wholeFileRange(key, offset, length);
|
|
5675
|
+
}
|
|
5676
|
+
}
|
|
5677
|
+
if (blocks.generationOf(key) !== view.generation) return wholeFileRange(key, offset, length);
|
|
5678
|
+
for (const [n, bytes] of fetched) blocks.keep(key, view.generation, n, bytes);
|
|
5679
|
+
return assembled(key, covered, fetched, offset, length);
|
|
5680
|
+
}
|
|
5681
|
+
return {
|
|
5682
|
+
async readRange(key, offset, length) {
|
|
5683
|
+
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || offset < 0 || length < 0) {
|
|
5684
|
+
throw endsBefore(key, offset, length);
|
|
5685
|
+
}
|
|
5686
|
+
return blockRange(key, offset, length);
|
|
5687
|
+
},
|
|
5688
|
+
async byteLength(key) {
|
|
5689
|
+
const view = blocks.viewOf(key);
|
|
5690
|
+
if (view.layout?.complete === true) return sizeOfBlocks(view.layout.blocks);
|
|
5691
|
+
const extended = await extendTo(key, view, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY);
|
|
5692
|
+
if (blocks.generationOf(key) !== view.generation) return (await readContent(key))?.byteLength ?? null;
|
|
5693
|
+
return extended === null ? null : sizeOfBlocks(extended.blocks);
|
|
5694
|
+
}
|
|
5695
|
+
};
|
|
5696
|
+
}
|
|
5697
|
+
|
|
5698
|
+
// src/persistence/durability/indexeddb-directory/index.ts
|
|
5699
|
+
function noValue() {
|
|
5700
|
+
return void 0;
|
|
5701
|
+
}
|
|
5702
|
+
function requireKey(key) {
|
|
5703
|
+
if (typeof key !== "string" || key.length === 0) {
|
|
5704
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
5705
|
+
}
|
|
5706
|
+
if (key.includes("\0")) {
|
|
5707
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
|
|
5708
|
+
}
|
|
5709
|
+
}
|
|
5710
|
+
function createIndexedDBDirectory(location, options) {
|
|
5711
|
+
const saveFailed = ErrorCodes.PERSISTENCE_SAVE_FAILED;
|
|
5712
|
+
const loadFailed = ErrorCodes.PERSISTENCE_LOAD_FAILED;
|
|
5713
|
+
const connection = openLogConnection(location);
|
|
5714
|
+
const blocks = createBlockMap();
|
|
5715
|
+
const openFiles = /* @__PURE__ */ new Map();
|
|
5716
|
+
const opening = /* @__PURE__ */ new Map();
|
|
5717
|
+
async function settledWrite(key, write, lengthsAfter) {
|
|
5718
|
+
const epoch = blocks.begin(key);
|
|
5719
|
+
let result;
|
|
5720
|
+
try {
|
|
5721
|
+
result = await write();
|
|
5722
|
+
} catch (error) {
|
|
5723
|
+
blocks.settle(key, epoch, void 0);
|
|
5724
|
+
throw error;
|
|
5725
|
+
}
|
|
5726
|
+
blocks.settle(key, epoch, lengthsAfter(result));
|
|
5727
|
+
return result;
|
|
5728
|
+
}
|
|
5729
|
+
function detach(key) {
|
|
5730
|
+
const file = openFiles.get(key);
|
|
5731
|
+
if (file === void 0) return;
|
|
5732
|
+
file.detached = true;
|
|
5733
|
+
openFiles.delete(key);
|
|
5734
|
+
}
|
|
5735
|
+
function release(file) {
|
|
5736
|
+
if (openFiles.get(file.key) === file) openFiles.delete(file.key);
|
|
5737
|
+
}
|
|
5738
|
+
async function openFile(key) {
|
|
5739
|
+
const existing = openFiles.get(key);
|
|
5740
|
+
if (existing !== void 0) return existing;
|
|
5741
|
+
const pending = opening.get(key);
|
|
5742
|
+
if (pending !== void 0) return pending;
|
|
5743
|
+
const opened = settledWrite(
|
|
5744
|
+
key,
|
|
5745
|
+
() => openFileRecords(connection, key),
|
|
5746
|
+
(file) => file.parts
|
|
5747
|
+
).then(
|
|
5748
|
+
(file) => {
|
|
5749
|
+
opening.delete(key);
|
|
5750
|
+
openFiles.set(key, file);
|
|
5751
|
+
return file;
|
|
5752
|
+
},
|
|
5753
|
+
(error) => {
|
|
5754
|
+
opening.delete(key);
|
|
5755
|
+
throw error;
|
|
5756
|
+
}
|
|
5757
|
+
);
|
|
5758
|
+
opening.set(key, opened);
|
|
5759
|
+
return opened;
|
|
5760
|
+
}
|
|
5761
|
+
async function readContent(key) {
|
|
5762
|
+
requireKey(key);
|
|
5763
|
+
const view = blocks.viewOf(key);
|
|
5764
|
+
const records = await connection.read(
|
|
5765
|
+
{ code: loadFailed, action: `read "${key}"` },
|
|
5766
|
+
(store) => readRecords(store, key)
|
|
5767
|
+
);
|
|
5768
|
+
if (records === null) return null;
|
|
5769
|
+
blocks.install(key, view, laidOut(lengthsOf(records)), true);
|
|
5770
|
+
return contentOf(records);
|
|
5771
|
+
}
|
|
5772
|
+
const ranges = createRangeReads(connection, blocks, readContent);
|
|
5773
|
+
function renameAfterMove(moving, from, to) {
|
|
5774
|
+
moving.key = to;
|
|
5775
|
+
if (openFiles.get(from) !== moving) return;
|
|
5776
|
+
openFiles.delete(from);
|
|
5777
|
+
detach(to);
|
|
5778
|
+
openFiles.set(to, moving);
|
|
5779
|
+
}
|
|
5780
|
+
async function moveRecords(from, to) {
|
|
5781
|
+
const fromEpoch = blocks.begin(from);
|
|
5782
|
+
const moved = settledWrite(
|
|
5783
|
+
to,
|
|
5784
|
+
() => connection.write("strict", { code: saveFailed, action: `move "${from}" to "${to}"` }, async (store) => {
|
|
5785
|
+
const records = await readRecords(store, from);
|
|
5786
|
+
if (records === null) {
|
|
5787
|
+
throw new NarsilError(saveFailed, `Failed to move "${from}" to "${to}": "${from}" holds nothing`, {
|
|
5788
|
+
from,
|
|
5789
|
+
to
|
|
5790
|
+
});
|
|
5791
|
+
}
|
|
5792
|
+
await replaceRecords(store, to, records);
|
|
5793
|
+
await removeRecords(store, from);
|
|
5794
|
+
return lengthsOf(records);
|
|
5795
|
+
}),
|
|
5796
|
+
(lengths) => lengths
|
|
5797
|
+
);
|
|
5798
|
+
const moving = openFiles.get(from);
|
|
5799
|
+
if (moving !== void 0) moving.moving = moved.then(noValue, noValue);
|
|
5800
|
+
try {
|
|
5801
|
+
await moved;
|
|
5802
|
+
if (moving !== void 0) renameAfterMove(moving, from, to);
|
|
5803
|
+
} finally {
|
|
5804
|
+
blocks.settle(from, fromEpoch, void 0);
|
|
5805
|
+
if (moving !== void 0) moving.moving = null;
|
|
5806
|
+
}
|
|
5807
|
+
}
|
|
5808
|
+
return {
|
|
5809
|
+
root: connection.root,
|
|
5810
|
+
async appendHandle(key) {
|
|
5811
|
+
requireKey(key);
|
|
5812
|
+
const file = await openFile(key);
|
|
5813
|
+
return createAppendHandle(file, { connection, blocks, relaxedAppends: options.relaxedAppends, release });
|
|
5814
|
+
},
|
|
5815
|
+
async markerHandle(key) {
|
|
5816
|
+
requireKey(key);
|
|
5817
|
+
let current = await settledWrite(
|
|
5818
|
+
key,
|
|
5819
|
+
() => connection.write("strict", { code: saveFailed, action: `open "${key}"` }, async (store) => {
|
|
5820
|
+
const records = await readRecords(store, key);
|
|
5821
|
+
if (records !== null) return contentOf(records);
|
|
5822
|
+
const empty = new Uint8Array(0);
|
|
5823
|
+
await requestResult(store.put(empty, key));
|
|
5824
|
+
return empty;
|
|
5825
|
+
}),
|
|
5826
|
+
(bytes) => [{ n: 0, length: bytes.byteLength }]
|
|
5827
|
+
);
|
|
5828
|
+
return {
|
|
5829
|
+
read: async () => current.slice(),
|
|
5830
|
+
async writeSlot(offset, slot) {
|
|
5831
|
+
const next = new Uint8Array(Math.max(current.byteLength, offset + slot.byteLength));
|
|
5832
|
+
next.set(current);
|
|
5833
|
+
next.set(slot, offset);
|
|
5834
|
+
await settledWrite(
|
|
5835
|
+
key,
|
|
5836
|
+
() => connection.write(
|
|
5837
|
+
"strict",
|
|
5838
|
+
{ code: ErrorCodes.PERSISTENCE_FSYNC_FAILED, action: `commit "${key}"` },
|
|
5839
|
+
(store) => replaceRecords(store, key, { first: next, appended: [] })
|
|
5840
|
+
),
|
|
5841
|
+
() => [{ n: 0, length: next.byteLength }]
|
|
5842
|
+
);
|
|
5843
|
+
current = next;
|
|
5844
|
+
},
|
|
5845
|
+
close: async () => void 0
|
|
5846
|
+
};
|
|
5847
|
+
},
|
|
5848
|
+
async atomicWrite(key, data) {
|
|
5849
|
+
requireKey(key);
|
|
5850
|
+
const pieces = splitIntoBlocks(Array.isArray(data) ? data : [data], INDEXEDDB_BLOCK_BYTES);
|
|
5851
|
+
const records = {
|
|
5852
|
+
first: pieces[0],
|
|
5853
|
+
appended: pieces.slice(1).map((bytes, at) => ({ n: at + 1, bytes }))
|
|
5854
|
+
};
|
|
5855
|
+
detach(key);
|
|
5856
|
+
await settledWrite(
|
|
5857
|
+
key,
|
|
5858
|
+
() => connection.write(
|
|
5859
|
+
"strict",
|
|
5860
|
+
{ code: saveFailed, action: `write "${key}"` },
|
|
5861
|
+
(store) => replaceRecords(store, key, records)
|
|
5862
|
+
),
|
|
5863
|
+
() => lengthsOf(records)
|
|
5864
|
+
);
|
|
5865
|
+
},
|
|
5866
|
+
async syncDirectoryOf(key) {
|
|
5867
|
+
requireKey(key);
|
|
5868
|
+
},
|
|
5869
|
+
read: readContent,
|
|
5870
|
+
async readRange(key, offset, length) {
|
|
5871
|
+
requireKey(key);
|
|
5872
|
+
return ranges.readRange(key, offset, length);
|
|
5873
|
+
},
|
|
5874
|
+
async byteLength(key) {
|
|
5875
|
+
requireKey(key);
|
|
5876
|
+
return ranges.byteLength(key);
|
|
5877
|
+
},
|
|
5878
|
+
async remove(key) {
|
|
5879
|
+
requireKey(key);
|
|
5880
|
+
detach(key);
|
|
5881
|
+
await settledWrite(
|
|
5882
|
+
key,
|
|
5883
|
+
() => connection.write(
|
|
5884
|
+
"strict",
|
|
5885
|
+
{ code: ErrorCodes.PERSISTENCE_DELETE_FAILED, action: `delete "${key}"` },
|
|
5886
|
+
(store) => removeRecords(store, key)
|
|
5887
|
+
),
|
|
5888
|
+
() => void 0
|
|
5889
|
+
);
|
|
5890
|
+
},
|
|
5891
|
+
async move(from, to) {
|
|
5892
|
+
requireKey(from);
|
|
5893
|
+
requireKey(to);
|
|
5894
|
+
if (from === to) return;
|
|
5895
|
+
detach(to);
|
|
5896
|
+
await moveRecords(from, to);
|
|
5897
|
+
},
|
|
5898
|
+
async list(prefix) {
|
|
5899
|
+
const keys = await connection.read(
|
|
5900
|
+
{ code: loadFailed, action: `list "${prefix}"` },
|
|
5901
|
+
(store) => listKeys(store, prefix)
|
|
5902
|
+
);
|
|
5903
|
+
return keys.sort(compareCodePoints);
|
|
5904
|
+
},
|
|
5905
|
+
identity: () => connection.identity(),
|
|
5906
|
+
async lock() {
|
|
5907
|
+
const held = await lockIndexedDBStore(location);
|
|
5908
|
+
return {
|
|
5909
|
+
holds: () => held.holds(),
|
|
5910
|
+
async release() {
|
|
5911
|
+
connection.close();
|
|
5912
|
+
await held.release();
|
|
5913
|
+
}
|
|
5914
|
+
};
|
|
5915
|
+
}
|
|
5916
|
+
};
|
|
5917
|
+
}
|
|
5918
|
+
|
|
4986
5919
|
// src/persistence/durability/snapshot-only/index-state.ts
|
|
4987
5920
|
function createSnapshotIndexState() {
|
|
4988
5921
|
return {
|
|
@@ -5069,7 +6002,8 @@ async function writeCaptured(input, store, prior, captured) {
|
|
|
5069
6002
|
await input.onCommitted({
|
|
5070
6003
|
saved: committed.saved,
|
|
5071
6004
|
documentCount: captured.documentCount,
|
|
5072
|
-
partitionCount: captured.partitionCount
|
|
6005
|
+
partitionCount: captured.partitionCount,
|
|
6006
|
+
positions: captured.positions
|
|
5073
6007
|
});
|
|
5074
6008
|
const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
|
|
5075
6009
|
await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
|
|
@@ -5091,14 +6025,13 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
5091
6025
|
});
|
|
5092
6026
|
try {
|
|
5093
6027
|
await writeCaptured(input, store, prior, captured);
|
|
5094
|
-
return { positions: captured.positions, partitionCount: captured.partitionCount };
|
|
5095
6028
|
} finally {
|
|
5096
6029
|
for (const plan of captured.plans.values()) plan.release();
|
|
5097
6030
|
}
|
|
5098
6031
|
}
|
|
5099
6032
|
|
|
5100
6033
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
5101
|
-
function
|
|
6034
|
+
function toError4(error) {
|
|
5102
6035
|
return error instanceof Error ? error : new Error(String(error));
|
|
5103
6036
|
}
|
|
5104
6037
|
function createPendingRestores(onFatalError) {
|
|
@@ -5119,7 +6052,7 @@ function createPendingRestores(onFatalError) {
|
|
|
5119
6052
|
return running;
|
|
5120
6053
|
};
|
|
5121
6054
|
pending.set(indexName, finish);
|
|
5122
|
-
void finish().catch((error) => onFatalError(
|
|
6055
|
+
void finish().catch((error) => onFatalError(toError4(error)));
|
|
5123
6056
|
return Promise.resolve();
|
|
5124
6057
|
},
|
|
5125
6058
|
async settle(indexName) {
|
|
@@ -5185,6 +6118,41 @@ function createSnapshotSavedSize(adapter, settleRestore) {
|
|
|
5185
6118
|
};
|
|
5186
6119
|
}
|
|
5187
6120
|
|
|
6121
|
+
// src/persistence/durability/snapshot-only/shutdown-save.ts
|
|
6122
|
+
function asError(error) {
|
|
6123
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
6124
|
+
}
|
|
6125
|
+
async function saveIndex(deps, indexName, indexState) {
|
|
6126
|
+
await indexState.applyChain;
|
|
6127
|
+
await checkpointInTurn(indexState, {
|
|
6128
|
+
joinsInFlight: false,
|
|
6129
|
+
mayStart: () => !indexState.unloading && indexState.mutationsSinceCheckpoint > 0,
|
|
6130
|
+
perform: () => indexState.applyChain.then(() => deps.checkpoint(indexName, indexState)),
|
|
6131
|
+
afterSettling: () => void 0
|
|
6132
|
+
});
|
|
6133
|
+
}
|
|
6134
|
+
async function saveUnsavedIndexes(deps) {
|
|
6135
|
+
let failure = null;
|
|
6136
|
+
for (const [indexName, indexState] of deps.indexes) {
|
|
6137
|
+
try {
|
|
6138
|
+
await saveIndex(deps, indexName, indexState);
|
|
6139
|
+
} catch (error) {
|
|
6140
|
+
const failed = asError(error);
|
|
6141
|
+
deps.onFailure(failed);
|
|
6142
|
+
failure ??= failed;
|
|
6143
|
+
}
|
|
6144
|
+
}
|
|
6145
|
+
if (failure !== null) throw failure;
|
|
6146
|
+
}
|
|
6147
|
+
async function saveAndReleaseIndexes(deps) {
|
|
6148
|
+
try {
|
|
6149
|
+
await saveUnsavedIndexes(deps);
|
|
6150
|
+
} finally {
|
|
6151
|
+
await deps.settleMetadataWrites();
|
|
6152
|
+
deps.release();
|
|
6153
|
+
}
|
|
6154
|
+
}
|
|
6155
|
+
|
|
5188
6156
|
// src/persistence/durability/snapshot-only/index.ts
|
|
5189
6157
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
5190
6158
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
@@ -5234,7 +6202,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
5234
6202
|
const manager = hooks.getManager(indexName);
|
|
5235
6203
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
5236
6204
|
if (manager === void 0 || metadata === void 0) return;
|
|
5237
|
-
|
|
6205
|
+
await writeSnapshotOnlyCheckpoint({
|
|
5238
6206
|
adapter,
|
|
5239
6207
|
indexName,
|
|
5240
6208
|
metadata,
|
|
@@ -5243,12 +6211,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
5243
6211
|
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
5244
6212
|
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
|
|
5245
6213
|
onCommitted: async (committed) => {
|
|
6214
|
+
noteSnapshotCommitted(indexState, committed.positions, committed.partitionCount);
|
|
5246
6215
|
savedSize.record(indexName, committed.saved);
|
|
5247
6216
|
hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
|
|
5248
6217
|
await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
|
|
5249
6218
|
}
|
|
5250
6219
|
});
|
|
5251
|
-
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
5252
6220
|
if (publisher !== void 0) {
|
|
5253
6221
|
const partitionIds = [];
|
|
5254
6222
|
for (let i = 0; i < manager.partitionCount; i += 1) {
|
|
@@ -5339,11 +6307,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
5339
6307
|
}
|
|
5340
6308
|
}
|
|
5341
6309
|
async function recordMutation(record) {
|
|
6310
|
+
if (shuttingDown) throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
5342
6311
|
const indexState = getOrCreateIndexState(record.indexName);
|
|
5343
6312
|
const buffered = indexState.applyChain.then(async () => {
|
|
5344
6313
|
await record.apply();
|
|
5345
6314
|
const appliedSeqNo2 = (indexState.appliedSeqNoByPartition.get(record.partitionId) ?? 0) + 1;
|
|
5346
6315
|
indexState.appliedSeqNoByPartition.set(record.partitionId, appliedSeqNo2);
|
|
6316
|
+
indexState.mutationsSinceCheckpoint += 1;
|
|
5347
6317
|
return appliedSeqNo2;
|
|
5348
6318
|
});
|
|
5349
6319
|
indexState.applyChain = buffered.then(
|
|
@@ -5351,7 +6321,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
5351
6321
|
() => void 0
|
|
5352
6322
|
);
|
|
5353
6323
|
const appliedSeqNo = await buffered;
|
|
5354
|
-
indexState.mutationsSinceCheckpoint += 1;
|
|
5355
6324
|
startCheckpointTimer();
|
|
5356
6325
|
if (indexState.mutationsSinceCheckpoint >= checkpointMutationThreshold) {
|
|
5357
6326
|
void checkpointIndex(record.indexName).catch((err) => {
|
|
@@ -5486,12 +6455,13 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
5486
6455
|
checkpointTimer = null;
|
|
5487
6456
|
}
|
|
5488
6457
|
await restores.settleAll();
|
|
5489
|
-
|
|
5490
|
-
|
|
5491
|
-
|
|
5492
|
-
|
|
5493
|
-
|
|
5494
|
-
|
|
6458
|
+
await saveAndReleaseIndexes({
|
|
6459
|
+
indexes,
|
|
6460
|
+
checkpoint: (indexName, indexState) => performCheckpoint(indexName, indexState, false),
|
|
6461
|
+
onFailure: hooks.onFatalError,
|
|
6462
|
+
settleMetadataWrites: () => metadataWrites.settleAll(),
|
|
6463
|
+
release: () => indexes.clear()
|
|
6464
|
+
});
|
|
5495
6465
|
}
|
|
5496
6466
|
};
|
|
5497
6467
|
}
|
|
@@ -5538,7 +6508,11 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
5538
6508
|
function readerOf(indexName) {
|
|
5539
6509
|
return hooks.previousDocumentReader === void 0 ? null : hooks.previousDocumentReader(indexName);
|
|
5540
6510
|
}
|
|
5541
|
-
const manager = tier.kind === "wal" ? createDurabilityManager(
|
|
6511
|
+
const manager = tier.kind === "wal" ? createDurabilityManager(
|
|
6512
|
+
tier.config,
|
|
6513
|
+
managerHooks,
|
|
6514
|
+
tier.indexedDB === void 0 ? void 0 : createIndexedDBDirectory(tier.indexedDB, { relaxedAppends: tier.config.mode === "async" })
|
|
6515
|
+
) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
5542
6516
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
5543
6517
|
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
5544
6518
|
const partitionId = fnv1a(docId) % partitionCount;
|
|
@@ -5656,7 +6630,7 @@ function createInvalidationIntegration(deps) {
|
|
|
5656
6630
|
const reloadChains = /* @__PURE__ */ new Map();
|
|
5657
6631
|
let broadcastTimer = null;
|
|
5658
6632
|
let stopped = false;
|
|
5659
|
-
function
|
|
6633
|
+
function toError5(err) {
|
|
5660
6634
|
return err instanceof Error ? err : new Error(String(err));
|
|
5661
6635
|
}
|
|
5662
6636
|
function pruneStaleForeignStats(entries) {
|
|
@@ -5669,7 +6643,7 @@ function createInvalidationIntegration(deps) {
|
|
|
5669
6643
|
}
|
|
5670
6644
|
function scheduleReload(indexName) {
|
|
5671
6645
|
const chain = reloadChains.get(indexName) ?? Promise.resolve();
|
|
5672
|
-
const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(
|
|
6646
|
+
const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(toError5(err)));
|
|
5673
6647
|
reloadChains.set(indexName, next);
|
|
5674
6648
|
}
|
|
5675
6649
|
function storeForeignStats(indexName, sourceInstanceId, stats) {
|
|
@@ -5695,7 +6669,7 @@ function createInvalidationIntegration(deps) {
|
|
|
5695
6669
|
}
|
|
5696
6670
|
storeForeignStats(event.indexName, event.instanceId, event.stats);
|
|
5697
6671
|
} catch (err) {
|
|
5698
|
-
onError(
|
|
6672
|
+
onError(toError5(err));
|
|
5699
6673
|
}
|
|
5700
6674
|
}
|
|
5701
6675
|
async function publishOwnStatistics() {
|
|
@@ -5717,7 +6691,7 @@ function createInvalidationIntegration(deps) {
|
|
|
5717
6691
|
}
|
|
5718
6692
|
});
|
|
5719
6693
|
} catch (err) {
|
|
5720
|
-
onError(
|
|
6694
|
+
onError(toError5(err));
|
|
5721
6695
|
}
|
|
5722
6696
|
}
|
|
5723
6697
|
}
|
|
@@ -5736,7 +6710,7 @@ function createInvalidationIntegration(deps) {
|
|
|
5736
6710
|
sourceInstanceId: instanceId
|
|
5737
6711
|
});
|
|
5738
6712
|
} catch (err) {
|
|
5739
|
-
onError(
|
|
6713
|
+
onError(toError5(err));
|
|
5740
6714
|
}
|
|
5741
6715
|
},
|
|
5742
6716
|
broadcastStats(indexName, tokens) {
|
|
@@ -5786,7 +6760,7 @@ function createInvalidationIntegration(deps) {
|
|
|
5786
6760
|
try {
|
|
5787
6761
|
await adapter.shutdown();
|
|
5788
6762
|
} catch (err) {
|
|
5789
|
-
onError(
|
|
6763
|
+
onError(toError5(err));
|
|
5790
6764
|
}
|
|
5791
6765
|
}
|
|
5792
6766
|
};
|