@delali/narsil 0.3.1-next.181 → 0.3.1-next.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
- package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
- package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
- package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
- package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
- package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
- package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
- package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
- package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
- package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
- package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
- package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
- package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
- package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +391 -138
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +1 -1
- package/dist/index.react-native.mjs +1 -1
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
- package/dist/query-suggestions/merge-worker.mjs +4 -4
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +2 -2
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
- package/dist/react-native/partial-words.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +1 -1
- package/dist/react-native/sync.mjs +1 -1
- package/dist/react-native/synonyms.mjs +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +16 -16
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +1 -1
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +1 -1
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/openapi.json +32 -0
- package/package.json +7 -7
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES
|
|
2
|
-
export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-
|
|
3
|
-
import {
|
|
4
|
-
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange,
|
|
5
|
-
import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-
|
|
6
|
-
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-
|
|
7
|
-
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-
|
|
8
|
-
export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-
|
|
1
|
+
import { scheduleSyncRetention, createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-M3H7CTIC.mjs';
|
|
2
|
+
export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-M3H7CTIC.mjs';
|
|
3
|
+
import { buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, metadataConfigOf, durableMetadataConfigOf, chunksOf, isRecord, nestedSchemaOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, requireSavedPartitionCount, createDetachedIndex, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readSegmentManifest, readMetadataEnvelope, snapshotCheckpointFor, whileGraphsStayComplete, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, highestPartitionIdOf, growPartitionsTo, createMarkerWriter } from './chunk-ZVZ2V2PZ.mjs';
|
|
4
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, requireSavedPartitionCount, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-ZVZ2V2PZ.mjs';
|
|
5
|
+
import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-CKS4QQEC.mjs';
|
|
6
|
+
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-CKS4QQEC.mjs';
|
|
7
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-SA7U67XF.mjs';
|
|
8
|
+
export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-SA7U67XF.mjs';
|
|
9
9
|
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-X4UMDSB4.mjs';
|
|
10
|
-
import { readSnapshotArchive, decodeSegmentManifest, isRecord as isRecord$1, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-
|
|
11
|
-
export { readSnapshotArchive } from './chunk-
|
|
12
|
-
import { createDurableDirectory } from './chunk-
|
|
13
|
-
import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, readSegmentFile, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-
|
|
14
|
-
export { checkPagedContents, checkPagedSelections, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-
|
|
15
|
-
import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-
|
|
10
|
+
import { readSnapshotArchive, decodeSegmentManifest, isRecord as isRecord$1, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-NPXORGMK.mjs';
|
|
11
|
+
export { readSnapshotArchive } from './chunk-NPXORGMK.mjs';
|
|
12
|
+
import { createDurableDirectory } from './chunk-6ICOWB6I.mjs';
|
|
13
|
+
import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, openPagedSegmentScan, countsAsLatestHolder, writePagedSegment, bytesThatSegmentsHoldInMemory, savedBytesOfManifest, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, checkPagedSegmentFully, readSegmentFile, NOTHING_SAVED, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, checkpointGarbage, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-3OEXOSWI.mjs';
|
|
14
|
+
export { bytesThatSegmentsHoldInMemory, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSegmentScan, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-3OEXOSWI.mjs';
|
|
15
|
+
import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-TTTBE4TB.mjs';
|
|
16
16
|
import './chunk-5FLRXF73.mjs';
|
|
17
17
|
import './chunk-JDDXV6UF.mjs';
|
|
18
|
-
import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-
|
|
18
|
+
import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-PK3BGGHO.mjs';
|
|
19
19
|
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
20
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-
|
|
20
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-MSIN4NM3.mjs';
|
|
21
21
|
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
22
22
|
export { bindStorageHost } from './chunk-CV3Q2EKO.mjs';
|
|
23
23
|
import { sha256 } from './chunk-LNGE7SLY.mjs';
|
|
24
24
|
import { setNestedValue } from './chunk-I2WJCRIH.mjs';
|
|
25
|
-
import { rewriteSpillOf } from './chunk-
|
|
26
|
-
import { deserializePartition } from './chunk-
|
|
25
|
+
import { rewriteSpillOf } from './chunk-YWSIRKKQ.mjs';
|
|
26
|
+
import { deserializePartition } from './chunk-OQANB7SJ.mjs';
|
|
27
27
|
import { decodeVectorIndexParts, decodeVectorFilePayload, isDead, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, graphPartsOfEarlierPayload, decodeVectorGraphPayload } from './chunk-TNEU32EN.mjs';
|
|
28
28
|
export { createHNSWIndex, osqBitsOf } from './chunk-TNEU32EN.mjs';
|
|
29
29
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
30
|
-
import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-
|
|
31
|
-
import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createPartitionRouter, buildSegmentPayloads, createRequestId } from './chunk-
|
|
30
|
+
import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
|
|
31
|
+
import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createPartitionRouter, buildSegmentPayloads, createRequestId } from './chunk-CWXR7OP4.mjs';
|
|
32
32
|
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
33
33
|
import './chunk-LPRRFCVU.mjs';
|
|
34
34
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
@@ -49,28 +49,55 @@ import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
|
49
49
|
function yieldToEventLoop() {
|
|
50
50
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
51
51
|
}
|
|
52
|
+
function newerTombstonesOf(prepared, at) {
|
|
53
|
+
const docIds = /* @__PURE__ */ new Set();
|
|
54
|
+
for (let newer = at + 1; newer < prepared.length; newer++) {
|
|
55
|
+
const access = prepared[newer].segment.paged;
|
|
56
|
+
if (access === null) continue;
|
|
57
|
+
for (const docId of access.snapshot.tombstones) docIds.add(docId);
|
|
58
|
+
}
|
|
59
|
+
return docIds;
|
|
60
|
+
}
|
|
61
|
+
function scansEveryId(segment, tombstoned) {
|
|
62
|
+
const held = segment.arrays.idTable.count;
|
|
63
|
+
return tombstoned.size * Math.ceil(Math.log2(held + 1)) > held;
|
|
64
|
+
}
|
|
65
|
+
function tombstoneScannedOrdinal(segment, tombstoned, ordinal) {
|
|
66
|
+
const docId = segment.arrays.idTable.idAt(ordinal);
|
|
67
|
+
if (tombstoned.has(docId)) segment.prepareTombstone(docId, void 0, void 0, ordinal)?.publish();
|
|
68
|
+
}
|
|
52
69
|
function applyNewerTombstones(prepared) {
|
|
53
70
|
for (let at = 0; at < prepared.length; at++) {
|
|
54
71
|
if (!prepared[at].isNew) continue;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
for (const docId of
|
|
72
|
+
const { segment } = prepared[at];
|
|
73
|
+
const tombstoned = newerTombstonesOf(prepared, at);
|
|
74
|
+
if (!scansEveryId(segment, tombstoned)) {
|
|
75
|
+
for (const docId of tombstoned) segment.tombstoneDocument(docId);
|
|
76
|
+
continue;
|
|
59
77
|
}
|
|
78
|
+
const held = segment.arrays.idTable.count;
|
|
79
|
+
for (let ordinal = 0; ordinal < held; ordinal++) tombstoneScannedOrdinal(segment, tombstoned, ordinal);
|
|
60
80
|
}
|
|
61
81
|
}
|
|
62
82
|
async function applyNewerTombstonesInTurns(prepared) {
|
|
63
|
-
let
|
|
83
|
+
let steps = 0;
|
|
64
84
|
for (let at = 0; at < prepared.length; at++) {
|
|
65
85
|
if (!prepared[at].isNew) continue;
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
for (const docId of
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
if (
|
|
86
|
+
const { segment } = prepared[at];
|
|
87
|
+
const tombstoned = newerTombstonesOf(prepared, at);
|
|
88
|
+
if (!scansEveryId(segment, tombstoned)) {
|
|
89
|
+
for (const docId of tombstoned) {
|
|
90
|
+
segment.tombstoneDocument(docId);
|
|
91
|
+
steps += 1;
|
|
92
|
+
if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
|
|
73
93
|
}
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
const held = segment.arrays.idTable.count;
|
|
97
|
+
for (let ordinal = 0; ordinal < held; ordinal++) {
|
|
98
|
+
tombstoneScannedOrdinal(segment, tombstoned, ordinal);
|
|
99
|
+
steps += 1;
|
|
100
|
+
if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
|
|
74
101
|
}
|
|
75
102
|
}
|
|
76
103
|
}
|
|
@@ -439,6 +466,9 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
439
466
|
return whole;
|
|
440
467
|
};
|
|
441
468
|
}
|
|
469
|
+
function liveDocumentsOf(manager, targets) {
|
|
470
|
+
return targets.map(({ partitionId }) => ({ partitionId, documentCount: manager.getPartition(partitionId).count() }));
|
|
471
|
+
}
|
|
442
472
|
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
443
473
|
const whileNoBuildPassOrMutationRuns = (capture2) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture2));
|
|
444
474
|
return whileNoBuildPassOrMutationRuns(
|
|
@@ -481,7 +511,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
|
|
|
481
511
|
vectorPlans,
|
|
482
512
|
wholePartitions,
|
|
483
513
|
recordsLeftInLog,
|
|
484
|
-
unreplayedCutoffs: cutoffs
|
|
514
|
+
unreplayedCutoffs: cutoffs,
|
|
515
|
+
liveDocuments: memoryMatchesTheTargets && cutoffs.size === 0 ? liveDocumentsOf(manager, targets) : null
|
|
485
516
|
};
|
|
486
517
|
} catch (error) {
|
|
487
518
|
for (const plan of vectorPlans.values()) plan.release();
|
|
@@ -702,7 +733,8 @@ function workerCanWriteEverySegment(input) {
|
|
|
702
733
|
}
|
|
703
734
|
async function writeIndexCheckpointSegments(input) {
|
|
704
735
|
const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
|
|
705
|
-
const { keepsWriteRecords } = input;
|
|
736
|
+
const { keepsWriteRecords, liveDocuments, rewriteBytesPerCheckpoint } = input;
|
|
737
|
+
const counted = liveDocuments === null ? {} : { liveDocuments, rewriteBytesPerCheckpoint };
|
|
706
738
|
const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
|
|
707
739
|
root: directory.root,
|
|
708
740
|
metadata,
|
|
@@ -710,7 +742,8 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
710
742
|
compactionThreshold,
|
|
711
743
|
prior,
|
|
712
744
|
capturedPartitions: [...capturedPartitions],
|
|
713
|
-
keepsWriteRecords
|
|
745
|
+
keepsWriteRecords,
|
|
746
|
+
...counted
|
|
714
747
|
}) : null;
|
|
715
748
|
if (offloaded !== null) {
|
|
716
749
|
return offloaded;
|
|
@@ -724,7 +757,8 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
724
757
|
prior,
|
|
725
758
|
wholePartitions,
|
|
726
759
|
capturedPartitions,
|
|
727
|
-
keepsWriteRecords
|
|
760
|
+
keepsWriteRecords,
|
|
761
|
+
...counted
|
|
728
762
|
});
|
|
729
763
|
}
|
|
730
764
|
|
|
@@ -921,23 +955,28 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
921
955
|
metadata,
|
|
922
956
|
targets,
|
|
923
957
|
compactionThreshold: input.compactionThreshold,
|
|
958
|
+
rewriteBytesPerCheckpoint: input.rewriteBytesPerCheckpoint,
|
|
924
959
|
canOffload: input.canOffload,
|
|
925
960
|
prior: priorForLayout(priorManifest, capture2.partitionCount),
|
|
926
961
|
wholePartitions: capture2.wholePartitions.contents,
|
|
927
962
|
capturedPartitions: capture2.wholePartitions.captured,
|
|
928
|
-
keepsWriteRecords: writeRecords !== null
|
|
963
|
+
keepsWriteRecords: writeRecords !== null,
|
|
964
|
+
liveDocuments: capture2.liveDocuments
|
|
929
965
|
})
|
|
930
966
|
);
|
|
931
967
|
await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
|
|
932
968
|
if (aTargetFailed(input.indexState, targets)) return 0;
|
|
933
969
|
const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
|
|
970
|
+
input.recordSavedBytes(written.saved);
|
|
934
971
|
input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
|
|
935
972
|
recordWrittenVectors(vectorIndexes, liveVectors.written);
|
|
936
973
|
await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
|
|
937
974
|
const changes = changesSinceCapture();
|
|
938
975
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
976
|
+
await writeCheckpointMetadata(input, documentCount, capture2.partitionCount, written.saved.bytes);
|
|
939
977
|
if (!input.holdsCheckpointFiles()) {
|
|
940
|
-
const
|
|
978
|
+
const garbage = await checkpointGarbage(store, input.indexName, written.manifest);
|
|
979
|
+
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, garbage);
|
|
941
980
|
const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
942
981
|
const keptForSync = await input.keepsCheckpointKey();
|
|
943
982
|
await removeCheckpointGarbage(
|
|
@@ -945,19 +984,6 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
945
984
|
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
|
|
946
985
|
);
|
|
947
986
|
}
|
|
948
|
-
const checkpointDocumentCount = documentCount;
|
|
949
|
-
await input.queueMetadataWrite(input.indexName, async () => {
|
|
950
|
-
const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
|
|
951
|
-
if (checkpointMetadata === void 0) {
|
|
952
|
-
return;
|
|
953
|
-
}
|
|
954
|
-
const bytes = await writeMetadataEnvelope(
|
|
955
|
-
{ ...checkpointMetadata, partitionCount: capture2.partitionCount },
|
|
956
|
-
{ checksum: true }
|
|
957
|
-
);
|
|
958
|
-
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
959
|
-
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture2.partitionCount);
|
|
960
|
-
});
|
|
961
987
|
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
962
988
|
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
963
989
|
if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
|
|
@@ -975,6 +1001,18 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
975
1001
|
const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
|
|
976
1002
|
return anotherIsDue ? eligibleBeyond : capture2.recordsLeftInLog;
|
|
977
1003
|
}
|
|
1004
|
+
async function writeCheckpointMetadata(input, documentCount, partitionCount, savedBytes) {
|
|
1005
|
+
const { indexName, hooks } = input;
|
|
1006
|
+
await input.queueMetadataWrite(indexName, async () => {
|
|
1007
|
+
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
|
|
1008
|
+
if (checkpointMetadata === void 0) {
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const bytes = await writeMetadataEnvelope({ ...checkpointMetadata, partitionCount }, { checksum: true });
|
|
1012
|
+
await input.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
1013
|
+
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, savedBytes);
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
978
1016
|
function recordsBeyond(indexState, targets, cutoffs) {
|
|
979
1017
|
let records = 0;
|
|
980
1018
|
for (const [partitionId, partition] of indexState.partitions) {
|
|
@@ -1051,6 +1089,258 @@ function createHeldCheckpoints(hooks) {
|
|
|
1051
1089
|
};
|
|
1052
1090
|
}
|
|
1053
1091
|
|
|
1092
|
+
// src/persistence/durability/index-stored-sizes.ts
|
|
1093
|
+
function refreshStoredSizes(sizes) {
|
|
1094
|
+
const known = sizes.saved !== null || !sizes.measuring && sizes.held;
|
|
1095
|
+
sizes.known = known;
|
|
1096
|
+
if (known) {
|
|
1097
|
+
sizes.savedBytes = sizes.saved?.bytes ?? 0;
|
|
1098
|
+
sizes.vectorSavedBytes = sizes.saved?.vectorBytes ?? 0;
|
|
1099
|
+
sizes.logBytes = sizes.tracksLog ? sizes.logged?.bytes ?? 0 : void 0;
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
sizes.savedBytes = sizes.measuring ? void 0 : sizes.recorded;
|
|
1103
|
+
sizes.vectorSavedBytes = void 0;
|
|
1104
|
+
sizes.logBytes = void 0;
|
|
1105
|
+
}
|
|
1106
|
+
function savedBytesByPartitionOf(saved) {
|
|
1107
|
+
const byPartition = [];
|
|
1108
|
+
if (saved === null) return byPartition;
|
|
1109
|
+
for (const { partitionId, bytes } of saved.partitions) byPartition[partitionId] = bytes;
|
|
1110
|
+
return byPartition;
|
|
1111
|
+
}
|
|
1112
|
+
function createTrackedSizes(tracksLog) {
|
|
1113
|
+
const sizes = {
|
|
1114
|
+
savedBytes: void 0,
|
|
1115
|
+
logBytes: void 0,
|
|
1116
|
+
vectorSavedBytes: void 0,
|
|
1117
|
+
saved: null,
|
|
1118
|
+
savedBytesByPartition: [],
|
|
1119
|
+
measuring: false,
|
|
1120
|
+
held: false,
|
|
1121
|
+
recorded: void 0,
|
|
1122
|
+
logged: null,
|
|
1123
|
+
known: false,
|
|
1124
|
+
tracksLog,
|
|
1125
|
+
addTo(stats) {
|
|
1126
|
+
if (!sizes.known) return;
|
|
1127
|
+
const saved = sizes.savedBytesByPartition;
|
|
1128
|
+
const logged = sizes.tracksLog ? sizes.logged?.bytesByPartition ?? [] : null;
|
|
1129
|
+
for (const partition of stats) {
|
|
1130
|
+
partition.savedBytes = saved[partition.partitionId] ?? 0;
|
|
1131
|
+
if (logged !== null) partition.logBytes = logged[partition.partitionId] ?? 0;
|
|
1132
|
+
}
|
|
1133
|
+
},
|
|
1134
|
+
recordSavedBytes(bytes) {
|
|
1135
|
+
sizes.recorded = bytes;
|
|
1136
|
+
refreshStoredSizes(sizes);
|
|
1137
|
+
}
|
|
1138
|
+
};
|
|
1139
|
+
return sizes;
|
|
1140
|
+
}
|
|
1141
|
+
function createStoredSizesRecord(tracksLog) {
|
|
1142
|
+
const sizesByIndex = /* @__PURE__ */ new Map();
|
|
1143
|
+
function of(indexName) {
|
|
1144
|
+
let sizes = sizesByIndex.get(indexName);
|
|
1145
|
+
if (sizes === void 0) {
|
|
1146
|
+
sizes = createTrackedSizes(tracksLog);
|
|
1147
|
+
sizesByIndex.set(indexName, sizes);
|
|
1148
|
+
}
|
|
1149
|
+
return sizes;
|
|
1150
|
+
}
|
|
1151
|
+
function forget(indexName) {
|
|
1152
|
+
const sizes = sizesByIndex.get(indexName);
|
|
1153
|
+
if (sizes === void 0) return;
|
|
1154
|
+
sizes.saved = null;
|
|
1155
|
+
sizes.savedBytesByPartition = [];
|
|
1156
|
+
sizes.measuring = false;
|
|
1157
|
+
sizes.held = false;
|
|
1158
|
+
refreshStoredSizes(sizes);
|
|
1159
|
+
}
|
|
1160
|
+
return {
|
|
1161
|
+
of,
|
|
1162
|
+
setSaved(indexName, saved) {
|
|
1163
|
+
const sizes = of(indexName);
|
|
1164
|
+
sizes.saved = saved;
|
|
1165
|
+
sizes.savedBytesByPartition = savedBytesByPartitionOf(saved);
|
|
1166
|
+
refreshStoredSizes(sizes);
|
|
1167
|
+
},
|
|
1168
|
+
setMeasuring(indexName, measuring) {
|
|
1169
|
+
const sizes = of(indexName);
|
|
1170
|
+
sizes.measuring = measuring;
|
|
1171
|
+
refreshStoredSizes(sizes);
|
|
1172
|
+
},
|
|
1173
|
+
hold(indexName) {
|
|
1174
|
+
const sizes = of(indexName);
|
|
1175
|
+
if (!sizes.held) {
|
|
1176
|
+
sizes.held = true;
|
|
1177
|
+
refreshStoredSizes(sizes);
|
|
1178
|
+
}
|
|
1179
|
+
return sizes;
|
|
1180
|
+
},
|
|
1181
|
+
forget,
|
|
1182
|
+
drop(indexName) {
|
|
1183
|
+
forget(indexName);
|
|
1184
|
+
sizesByIndex.delete(indexName);
|
|
1185
|
+
}
|
|
1186
|
+
};
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
// src/persistence/durability/log-bytes.ts
|
|
1190
|
+
var LOG_SEGMENT_KEY = /^(.+)\/wal\/(\d+)\/\d{16}$/;
|
|
1191
|
+
var HISTORY_SEGMENT_KEY = /^(.+)\/history\/(\d+)\/\d{16}-\d{16}$/;
|
|
1192
|
+
function loggedFileOf(key) {
|
|
1193
|
+
const match = LOG_SEGMENT_KEY.exec(key) ?? HISTORY_SEGMENT_KEY.exec(key);
|
|
1194
|
+
if (match === null) return null;
|
|
1195
|
+
const partitionId = Number.parseInt(match[2], 10);
|
|
1196
|
+
return Number.isSafeInteger(partitionId) ? { indexName: match[1], partitionId } : null;
|
|
1197
|
+
}
|
|
1198
|
+
function emptyLog() {
|
|
1199
|
+
return { files: /* @__PURE__ */ new Map(), bytes: 0, bytesByPartition: [], sizes: null };
|
|
1200
|
+
}
|
|
1201
|
+
function addBytes(log, partitionId, delta) {
|
|
1202
|
+
log.bytes += delta;
|
|
1203
|
+
log.bytesByPartition[partitionId] = (log.bytesByPartition[partitionId] ?? 0) + delta;
|
|
1204
|
+
if (log.sizes !== null) refreshStoredSizes(log.sizes);
|
|
1205
|
+
}
|
|
1206
|
+
function makeCurrent(previous, log, sizes) {
|
|
1207
|
+
if (previous !== void 0 && previous !== log) previous.sizes = null;
|
|
1208
|
+
log.sizes = sizes;
|
|
1209
|
+
sizes.logged = log;
|
|
1210
|
+
refreshStoredSizes(sizes);
|
|
1211
|
+
}
|
|
1212
|
+
function setFileBytes(log, key, partitionId, bytes) {
|
|
1213
|
+
const previous = log.files.get(key);
|
|
1214
|
+
if (previous !== void 0) addBytes(log, previous.partitionId, -previous.bytes);
|
|
1215
|
+
log.files.set(key, { partitionId, bytes });
|
|
1216
|
+
addBytes(log, partitionId, bytes);
|
|
1217
|
+
}
|
|
1218
|
+
function removeFile(log, key) {
|
|
1219
|
+
const previous = log.files.get(key);
|
|
1220
|
+
if (previous === void 0) return void 0;
|
|
1221
|
+
log.files.delete(key);
|
|
1222
|
+
addBytes(log, previous.partitionId, -previous.bytes);
|
|
1223
|
+
return previous;
|
|
1224
|
+
}
|
|
1225
|
+
function createLogBytesLedger(base, sizes) {
|
|
1226
|
+
const logs = /* @__PURE__ */ new Map();
|
|
1227
|
+
function logOf(indexName) {
|
|
1228
|
+
let log = logs.get(indexName);
|
|
1229
|
+
if (log === void 0) {
|
|
1230
|
+
log = emptyLog();
|
|
1231
|
+
logs.set(indexName, log);
|
|
1232
|
+
makeCurrent(void 0, log, sizes.of(indexName));
|
|
1233
|
+
}
|
|
1234
|
+
return log;
|
|
1235
|
+
}
|
|
1236
|
+
function track(key, bytes) {
|
|
1237
|
+
const file = loggedFileOf(key);
|
|
1238
|
+
if (file !== null) setFileBytes(logOf(file.indexName), key, file.partitionId, bytes);
|
|
1239
|
+
}
|
|
1240
|
+
function untrack(key) {
|
|
1241
|
+
const file = loggedFileOf(key);
|
|
1242
|
+
if (file === null) return void 0;
|
|
1243
|
+
const log = logs.get(file.indexName);
|
|
1244
|
+
return log === void 0 ? void 0 : removeFile(log, key);
|
|
1245
|
+
}
|
|
1246
|
+
async function currentLength(key) {
|
|
1247
|
+
return await base.byteLength?.(key) ?? (await base.read(key))?.byteLength ?? 0;
|
|
1248
|
+
}
|
|
1249
|
+
function trackedHandle(key, file, handle) {
|
|
1250
|
+
async function remeasure() {
|
|
1251
|
+
const log = logs.get(file.indexName);
|
|
1252
|
+
if (log?.files.has(key) === true) setFileBytes(log, key, file.partitionId, await handle.size());
|
|
1253
|
+
}
|
|
1254
|
+
return {
|
|
1255
|
+
async append(bytes) {
|
|
1256
|
+
try {
|
|
1257
|
+
await handle.append(bytes);
|
|
1258
|
+
} catch (error) {
|
|
1259
|
+
await remeasure().catch(() => void 0);
|
|
1260
|
+
throw error;
|
|
1261
|
+
}
|
|
1262
|
+
const log = logs.get(file.indexName);
|
|
1263
|
+
const entry = log?.files.get(key);
|
|
1264
|
+
if (log === void 0 || entry === void 0) return;
|
|
1265
|
+
entry.bytes += bytes.byteLength;
|
|
1266
|
+
addBytes(log, file.partitionId, bytes.byteLength);
|
|
1267
|
+
},
|
|
1268
|
+
sync: () => handle.sync(),
|
|
1269
|
+
async truncate(byteLength2) {
|
|
1270
|
+
try {
|
|
1271
|
+
await handle.truncate(byteLength2);
|
|
1272
|
+
} finally {
|
|
1273
|
+
await remeasure().catch(() => void 0);
|
|
1274
|
+
}
|
|
1275
|
+
},
|
|
1276
|
+
size: () => handle.size(),
|
|
1277
|
+
close: () => handle.close()
|
|
1278
|
+
};
|
|
1279
|
+
}
|
|
1280
|
+
const directory = {
|
|
1281
|
+
root: base.root,
|
|
1282
|
+
async appendHandle(key) {
|
|
1283
|
+
const handle = await base.appendHandle(key);
|
|
1284
|
+
const file = loggedFileOf(key);
|
|
1285
|
+
if (file === null) return handle;
|
|
1286
|
+
if (logs.get(file.indexName)?.files.has(key) !== true) track(key, await handle.size());
|
|
1287
|
+
return trackedHandle(key, file, handle);
|
|
1288
|
+
},
|
|
1289
|
+
markerHandle: (key) => base.markerHandle(key),
|
|
1290
|
+
async atomicWrite(key, data) {
|
|
1291
|
+
await base.atomicWrite(key, data);
|
|
1292
|
+
if (loggedFileOf(key) === null) return;
|
|
1293
|
+
let bytes = 0;
|
|
1294
|
+
for (const chunk of Array.isArray(data) ? data : [data]) bytes += chunk.byteLength;
|
|
1295
|
+
track(key, bytes);
|
|
1296
|
+
},
|
|
1297
|
+
syncDirectoryOf: (key) => base.syncDirectoryOf(key),
|
|
1298
|
+
read: (key) => base.read(key),
|
|
1299
|
+
async remove(key) {
|
|
1300
|
+
await base.remove(key);
|
|
1301
|
+
untrack(key);
|
|
1302
|
+
},
|
|
1303
|
+
async move(from, to) {
|
|
1304
|
+
await base.move(from, to);
|
|
1305
|
+
const moved = untrack(from);
|
|
1306
|
+
if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
|
|
1307
|
+
},
|
|
1308
|
+
list: (prefix) => base.list(prefix),
|
|
1309
|
+
pathOf: (key) => base.pathOf(key)
|
|
1310
|
+
};
|
|
1311
|
+
const { readRange, byteLength, identity, lock } = base;
|
|
1312
|
+
if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
|
|
1313
|
+
if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
|
|
1314
|
+
if (identity !== void 0) directory.identity = () => identity.call(base);
|
|
1315
|
+
if (lock !== void 0) directory.lock = () => lock.call(base);
|
|
1316
|
+
return {
|
|
1317
|
+
directory,
|
|
1318
|
+
async seed(indexName) {
|
|
1319
|
+
const log = emptyLog();
|
|
1320
|
+
for (const prefix of [`${indexName}/wal/`, `${indexName}/history/`]) {
|
|
1321
|
+
for (const key of await base.list(prefix)) {
|
|
1322
|
+
const file = loggedFileOf(key);
|
|
1323
|
+
if (file === null || file.indexName !== indexName) continue;
|
|
1324
|
+
setFileBytes(log, key, file.partitionId, await currentLength(key));
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
const previous = logs.get(indexName);
|
|
1328
|
+
logs.set(indexName, log);
|
|
1329
|
+
makeCurrent(previous, log, sizes.of(indexName));
|
|
1330
|
+
},
|
|
1331
|
+
forget(indexName) {
|
|
1332
|
+
const log = logs.get(indexName);
|
|
1333
|
+
if (log === void 0) return;
|
|
1334
|
+
logs.delete(indexName);
|
|
1335
|
+
if (log.sizes !== null) {
|
|
1336
|
+
log.sizes.logged = null;
|
|
1337
|
+
refreshStoredSizes(log.sizes);
|
|
1338
|
+
log.sizes = null;
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
};
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1054
1344
|
// src/persistence/durability/restore-journal.ts
|
|
1055
1345
|
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
1056
1346
|
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
@@ -1447,6 +1737,7 @@ function createIndexUnloading(deps) {
|
|
|
1447
1737
|
}
|
|
1448
1738
|
await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
|
|
1449
1739
|
deps.forgetSyncState(indexName);
|
|
1740
|
+
deps.dropStoredBytes(indexName);
|
|
1450
1741
|
},
|
|
1451
1742
|
async unloadIndex(indexName) {
|
|
1452
1743
|
const indexState = indexes.get(indexName);
|
|
@@ -1457,6 +1748,7 @@ function createIndexUnloading(deps) {
|
|
|
1457
1748
|
indexes.delete(indexName);
|
|
1458
1749
|
metadataWrites.forget(indexName);
|
|
1459
1750
|
deps.forgetSyncState(indexName);
|
|
1751
|
+
deps.forgetStoredBytes(indexName);
|
|
1460
1752
|
}
|
|
1461
1753
|
};
|
|
1462
1754
|
}
|
|
@@ -1920,7 +2212,7 @@ function createSyncStorage(deps) {
|
|
|
1920
2212
|
|
|
1921
2213
|
// src/persistence/durability/manager-storage.ts
|
|
1922
2214
|
function createIndexStorageAccess(deps) {
|
|
1923
|
-
const { directory,
|
|
2215
|
+
const { directory, logBytes, sizes } = deps;
|
|
1924
2216
|
const archiveAccess = createArchiveCheckpointAccess({
|
|
1925
2217
|
directory,
|
|
1926
2218
|
held: deps.heldCheckpointFiles,
|
|
@@ -1932,7 +2224,7 @@ function createIndexStorageAccess(deps) {
|
|
|
1932
2224
|
directory,
|
|
1933
2225
|
openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
|
|
1934
2226
|
inCheckpointTurn: (indexName, work) => {
|
|
1935
|
-
const indexState = indexes.get(indexName);
|
|
2227
|
+
const indexState = deps.indexes.get(indexName);
|
|
1936
2228
|
if (indexState === void 0) return work();
|
|
1937
2229
|
return checkpointInTurn(indexState, {
|
|
1938
2230
|
joinsInFlight: false,
|
|
@@ -1945,12 +2237,54 @@ function createIndexStorageAccess(deps) {
|
|
|
1945
2237
|
const unloading = createIndexUnloading({
|
|
1946
2238
|
directory,
|
|
1947
2239
|
hooks: deps.hooks,
|
|
1948
|
-
indexes,
|
|
2240
|
+
indexes: deps.indexes,
|
|
1949
2241
|
metadataWrites: deps.metadataWrites,
|
|
1950
2242
|
closeWriterReportingFailure: deps.closeWriterReportingFailure,
|
|
1951
|
-
forgetSyncState: syncStorage.forget
|
|
2243
|
+
forgetSyncState: syncStorage.forget,
|
|
2244
|
+
forgetStoredBytes: (indexName) => {
|
|
2245
|
+
sizes.forget(indexName);
|
|
2246
|
+
logBytes.forget(indexName);
|
|
2247
|
+
},
|
|
2248
|
+
dropStoredBytes: (indexName) => {
|
|
2249
|
+
sizes.drop(indexName);
|
|
2250
|
+
logBytes.forget(indexName);
|
|
2251
|
+
}
|
|
1952
2252
|
});
|
|
1953
|
-
|
|
2253
|
+
async function measureStoredBytes(indexName, saved) {
|
|
2254
|
+
sizes.setSaved(indexName, saved);
|
|
2255
|
+
await logBytes.seed(indexName);
|
|
2256
|
+
}
|
|
2257
|
+
const recoveryDeps = {
|
|
2258
|
+
directory,
|
|
2259
|
+
hooks: deps.hooks,
|
|
2260
|
+
compactionThreshold: deps.compactionThreshold,
|
|
2261
|
+
registerPartition: deps.registerPartition,
|
|
2262
|
+
holdsCheckpointFiles: (indexName) => deps.heldCheckpointFiles.has(indexName),
|
|
2263
|
+
queueMetadataWrite: deps.metadataWrites.queue,
|
|
2264
|
+
syncStorage,
|
|
2265
|
+
measureStoredBytes
|
|
2266
|
+
};
|
|
2267
|
+
const storedBytesAccess = {
|
|
2268
|
+
storedSizesOf: sizes.of,
|
|
2269
|
+
addStoredSizes(indexName, stats) {
|
|
2270
|
+
sizes.of(indexName).addTo(stats);
|
|
2271
|
+
},
|
|
2272
|
+
vectorSavedBytes(indexName) {
|
|
2273
|
+
return sizes.of(indexName).vectorSavedBytes;
|
|
2274
|
+
},
|
|
2275
|
+
async measureRestoredStorage(indexName, restored) {
|
|
2276
|
+
if (restored !== void 0) {
|
|
2277
|
+
await measureStoredBytes(indexName, restored);
|
|
2278
|
+
return;
|
|
2279
|
+
}
|
|
2280
|
+
const decoded = await readSegmentManifest(directoryCheckpointStore(directory), indexName);
|
|
2281
|
+
await measureStoredBytes(
|
|
2282
|
+
indexName,
|
|
2283
|
+
decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED
|
|
2284
|
+
);
|
|
2285
|
+
}
|
|
2286
|
+
};
|
|
2287
|
+
return { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps };
|
|
1954
2288
|
}
|
|
1955
2289
|
|
|
1956
2290
|
// src/persistence/durability/metadata-write-queue.ts
|
|
@@ -2476,7 +2810,7 @@ function createWalWriter(directory, config) {
|
|
|
2476
2810
|
}
|
|
2477
2811
|
|
|
2478
2812
|
// src/persistence/durability/partition-registry.ts
|
|
2479
|
-
function createPartitionRegistry(directory, segmentMaxBytes) {
|
|
2813
|
+
function createPartitionRegistry(directory, segmentMaxBytes, onIndexHeld) {
|
|
2480
2814
|
const indexes = /* @__PURE__ */ new Map();
|
|
2481
2815
|
function indexState(indexName) {
|
|
2482
2816
|
let state = indexes.get(indexName);
|
|
@@ -2490,6 +2824,7 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
|
|
|
2490
2824
|
stalledWrites: []
|
|
2491
2825
|
};
|
|
2492
2826
|
indexes.set(indexName, state);
|
|
2827
|
+
onIndexHeld(indexName);
|
|
2493
2828
|
}
|
|
2494
2829
|
return state;
|
|
2495
2830
|
}
|
|
@@ -3267,19 +3602,18 @@ async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
|
|
|
3267
3602
|
const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
|
|
3268
3603
|
recordWrittenVectors(deps.vectorIndexes, vectors.written);
|
|
3269
3604
|
await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
|
|
3270
|
-
await
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
);
|
|
3605
|
+
const garbage = await checkpointGarbage(deps.store, deps.indexName, committed.manifest);
|
|
3606
|
+
await removeCheckpointGarbage(deps.store, await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, garbage));
|
|
3607
|
+
return committed.saved;
|
|
3274
3608
|
}
|
|
3275
3609
|
async function upgradeEarlierLayout(deps, earlier) {
|
|
3276
3610
|
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
|
|
3277
3611
|
try {
|
|
3278
3612
|
const loaded = await loadEarlier(deps, earlier);
|
|
3279
3613
|
const targets = await deps.replay(loaded.checkpoint);
|
|
3280
|
-
await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
|
|
3614
|
+
const saved = await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
|
|
3281
3615
|
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
|
|
3282
|
-
return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
|
|
3616
|
+
return { checkpoint: targets, documentCount: deps.manager.countDocuments(), saved };
|
|
3283
3617
|
} catch (error) {
|
|
3284
3618
|
const failure = error instanceof Error ? error : new Error(String(error));
|
|
3285
3619
|
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
|
|
@@ -3299,11 +3633,11 @@ async function recoverCheckpoint(deps) {
|
|
|
3299
3633
|
const earlier = await earlierLayoutOf(deps, decoded);
|
|
3300
3634
|
if (earlier !== null) {
|
|
3301
3635
|
const upgraded = await upgradeEarlierLayout(deps, earlier);
|
|
3302
|
-
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
|
|
3636
|
+
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount, saved: upgraded.saved };
|
|
3303
3637
|
}
|
|
3304
3638
|
if (decoded === null || decoded.kind !== "current") {
|
|
3305
3639
|
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
|
|
3306
|
-
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
|
|
3640
|
+
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null, saved: NOTHING_SAVED };
|
|
3307
3641
|
}
|
|
3308
3642
|
await deps.store.remove(snapshotBundleKey(deps.indexName));
|
|
3309
3643
|
if (!deps.keepsUnlistedFiles) {
|
|
@@ -3316,7 +3650,11 @@ async function recoverCheckpoint(deps) {
|
|
|
3316
3650
|
vectorIndexes: deps.vectorIndexes,
|
|
3317
3651
|
writeRecords: deps.writeRecords ?? null
|
|
3318
3652
|
});
|
|
3319
|
-
return {
|
|
3653
|
+
return {
|
|
3654
|
+
checkpoint: await deps.replay(checkpoint),
|
|
3655
|
+
upgradedDocumentCount: null,
|
|
3656
|
+
saved: savedBytesOfManifest(decoded.manifest)
|
|
3657
|
+
};
|
|
3320
3658
|
}
|
|
3321
3659
|
|
|
3322
3660
|
// src/persistence/durability/layout-transition/recovery.ts
|
|
@@ -3418,9 +3756,9 @@ async function settleStoredAnalysis(store, metadata, metadataOnly) {
|
|
|
3418
3756
|
}
|
|
3419
3757
|
|
|
3420
3758
|
// src/persistence/durability/recover-index.ts
|
|
3421
|
-
async function writeCountedMetadata(deps, indexName, documentCount) {
|
|
3759
|
+
async function writeCountedMetadata(deps, indexName, documentCount, savedBytes) {
|
|
3422
3760
|
await deps.queueMetadataWrite(indexName, async () => {
|
|
3423
|
-
const metadata = deps.hooks.buildMetadata(indexName, documentCount);
|
|
3761
|
+
const metadata = deps.hooks.buildMetadata(indexName, documentCount, savedBytes);
|
|
3424
3762
|
if (metadata === void 0) return;
|
|
3425
3763
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
3426
3764
|
await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
@@ -3515,8 +3853,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3515
3853
|
for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
|
|
3516
3854
|
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
3517
3855
|
}
|
|
3856
|
+
await deps.measureStoredBytes(indexName, recovered.saved);
|
|
3518
3857
|
if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
|
|
3519
|
-
await writeCountedMetadata(
|
|
3858
|
+
await writeCountedMetadata(
|
|
3859
|
+
deps,
|
|
3860
|
+
indexName,
|
|
3861
|
+
recovered.upgradedDocumentCount ?? manager.countDocuments(),
|
|
3862
|
+
recovered.saved.bytes
|
|
3863
|
+
);
|
|
3520
3864
|
}
|
|
3521
3865
|
return layout.transition === null ? null : removeUnread;
|
|
3522
3866
|
}
|
|
@@ -3532,15 +3876,22 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
|
|
|
3532
3876
|
|
|
3533
3877
|
// src/persistence/durability/manager.ts
|
|
3534
3878
|
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
3535
|
-
const
|
|
3879
|
+
const sizes = createStoredSizesRecord(true);
|
|
3880
|
+
const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
|
|
3881
|
+
const { directory } = logBytes;
|
|
3536
3882
|
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
3537
3883
|
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
3538
3884
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
3539
3885
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
3540
3886
|
const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
|
|
3887
|
+
const rewriteBytesPerCheckpoint = config.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT;
|
|
3541
3888
|
const mode = config.mode ?? "sync";
|
|
3542
3889
|
const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
|
|
3543
|
-
const registry = createPartitionRegistry(
|
|
3890
|
+
const registry = createPartitionRegistry(
|
|
3891
|
+
directory,
|
|
3892
|
+
segmentMaxBytes,
|
|
3893
|
+
(indexName) => hooks.linkStoredSizes(indexName, sizes.hold(indexName))
|
|
3894
|
+
);
|
|
3544
3895
|
const indexes = registry.indexes;
|
|
3545
3896
|
const getOrCreateIndexState = registry.indexState;
|
|
3546
3897
|
const getOrCreatePartition = registry.partition;
|
|
@@ -3631,12 +3982,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3631
3982
|
indexName,
|
|
3632
3983
|
indexState,
|
|
3633
3984
|
compactionThreshold,
|
|
3985
|
+
rewriteBytesPerCheckpoint,
|
|
3634
3986
|
canOffload: canOffloadCheckpoint,
|
|
3635
3987
|
fromMemory,
|
|
3636
3988
|
mutationThreshold: checkpointMutationThreshold,
|
|
3637
3989
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
3638
3990
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
3639
3991
|
keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
|
|
3992
|
+
recordSavedBytes: (saved) => sizes.setSaved(indexName, saved),
|
|
3640
3993
|
queueMetadataWrite,
|
|
3641
3994
|
markFatal,
|
|
3642
3995
|
failPartition: partitionFailureFor(indexName, partitionBatchDeps)
|
|
@@ -3691,28 +4044,22 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3691
4044
|
startAsyncFlushTimer();
|
|
3692
4045
|
return outcomes;
|
|
3693
4046
|
}
|
|
3694
|
-
const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
|
|
4047
|
+
const { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps } = createIndexStorageAccess({
|
|
3695
4048
|
directory,
|
|
4049
|
+
logBytes,
|
|
4050
|
+
sizes,
|
|
3696
4051
|
hooks,
|
|
3697
4052
|
indexes,
|
|
3698
4053
|
metadataWrites,
|
|
3699
4054
|
heldCheckpointFiles,
|
|
3700
|
-
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3701
|
-
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
3702
|
-
markFatal,
|
|
3703
|
-
closeWriterReportingFailure
|
|
3704
|
-
});
|
|
3705
|
-
const recoveryDeps = {
|
|
3706
|
-
directory,
|
|
3707
|
-
hooks,
|
|
3708
4055
|
compactionThreshold,
|
|
4056
|
+
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3709
4057
|
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
|
|
3710
4058
|
getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
|
|
3711
4059
|
},
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
};
|
|
4060
|
+
markFatal,
|
|
4061
|
+
closeWriterReportingFailure
|
|
4062
|
+
});
|
|
3716
4063
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
3717
4064
|
const recovered = recoverWithLayoutCheckpoint(
|
|
3718
4065
|
recoveryDeps,
|
|
@@ -3809,6 +4156,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3809
4156
|
syncStorage: () => syncStorage,
|
|
3810
4157
|
...queryHistoryAccess({ kind: "directory", directory }),
|
|
3811
4158
|
...archiveAccess,
|
|
4159
|
+
...storedBytesAccess,
|
|
3812
4160
|
async shutdown() {
|
|
3813
4161
|
shuttingDown = true;
|
|
3814
4162
|
stopTimers();
|
|
@@ -3827,6 +4175,31 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3827
4175
|
};
|
|
3828
4176
|
}
|
|
3829
4177
|
|
|
4178
|
+
// src/persistence/durability/snapshot-only/index-state.ts
|
|
4179
|
+
function createSnapshotIndexState() {
|
|
4180
|
+
return {
|
|
4181
|
+
mutationsSinceCheckpoint: 0,
|
|
4182
|
+
checkpointInFlight: null,
|
|
4183
|
+
applyChain: Promise.resolve(),
|
|
4184
|
+
unloading: false,
|
|
4185
|
+
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
4186
|
+
};
|
|
4187
|
+
}
|
|
4188
|
+
function metadataKey(indexName) {
|
|
4189
|
+
return `${indexName}/meta`;
|
|
4190
|
+
}
|
|
4191
|
+
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
4192
|
+
let recordsLeft = 0;
|
|
4193
|
+
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
4194
|
+
if (partitionId >= partitionCount) {
|
|
4195
|
+
state.appliedSeqNoByPartition.delete(partitionId);
|
|
4196
|
+
continue;
|
|
4197
|
+
}
|
|
4198
|
+
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
4199
|
+
}
|
|
4200
|
+
state.mutationsSinceCheckpoint = recordsLeft;
|
|
4201
|
+
}
|
|
4202
|
+
|
|
3830
4203
|
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
3831
4204
|
function capture(input, prior, cutoffs) {
|
|
3832
4205
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
@@ -3885,10 +4258,21 @@ async function writeCaptured(input, store, prior, captured) {
|
|
|
3885
4258
|
});
|
|
3886
4259
|
const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
|
|
3887
4260
|
recordWrittenVectors(input.vectorIndexes, vectors.written);
|
|
3888
|
-
await
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
4261
|
+
await input.onCommitted({
|
|
4262
|
+
saved: committed.saved,
|
|
4263
|
+
documentCount: captured.documentCount,
|
|
4264
|
+
partitionCount: captured.partitionCount
|
|
4265
|
+
});
|
|
4266
|
+
const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
|
|
4267
|
+
await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
|
|
4268
|
+
}
|
|
4269
|
+
async function saveCheckpointMetadata(adapter, hooks, indexName, committed) {
|
|
4270
|
+
const { saved, documentCount, partitionCount } = committed;
|
|
4271
|
+
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, saved.bytes);
|
|
4272
|
+
if (checkpointMetadata === void 0) return;
|
|
4273
|
+
checkpointMetadata.partitionCount = partitionCount;
|
|
4274
|
+
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(checkpointMetadata, { checksum: true }));
|
|
4275
|
+
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, saved.bytes);
|
|
3892
4276
|
}
|
|
3893
4277
|
async function writeSnapshotOnlyCheckpoint(input) {
|
|
3894
4278
|
const store = adapterCheckpointStore(input.adapter);
|
|
@@ -3899,41 +4283,12 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
3899
4283
|
});
|
|
3900
4284
|
try {
|
|
3901
4285
|
await writeCaptured(input, store, prior, captured);
|
|
3902
|
-
return {
|
|
3903
|
-
documentCount: captured.documentCount,
|
|
3904
|
-
positions: captured.positions,
|
|
3905
|
-
partitionCount: captured.partitionCount
|
|
3906
|
-
};
|
|
4286
|
+
return { positions: captured.positions, partitionCount: captured.partitionCount };
|
|
3907
4287
|
} finally {
|
|
3908
4288
|
for (const plan of captured.plans.values()) plan.release();
|
|
3909
4289
|
}
|
|
3910
4290
|
}
|
|
3911
4291
|
|
|
3912
|
-
// src/persistence/durability/snapshot-only/index-state.ts
|
|
3913
|
-
function createSnapshotIndexState() {
|
|
3914
|
-
return {
|
|
3915
|
-
mutationsSinceCheckpoint: 0,
|
|
3916
|
-
checkpointInFlight: null,
|
|
3917
|
-
applyChain: Promise.resolve(),
|
|
3918
|
-
unloading: false,
|
|
3919
|
-
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
3920
|
-
};
|
|
3921
|
-
}
|
|
3922
|
-
function metadataKey(indexName) {
|
|
3923
|
-
return `${indexName}/meta`;
|
|
3924
|
-
}
|
|
3925
|
-
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
3926
|
-
let recordsLeft = 0;
|
|
3927
|
-
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
3928
|
-
if (partitionId >= partitionCount) {
|
|
3929
|
-
state.appliedSeqNoByPartition.delete(partitionId);
|
|
3930
|
-
continue;
|
|
3931
|
-
}
|
|
3932
|
-
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
3933
|
-
}
|
|
3934
|
-
state.mutationsSinceCheckpoint = recordsLeft;
|
|
3935
|
-
}
|
|
3936
|
-
|
|
3937
4292
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
3938
4293
|
function toError5(error) {
|
|
3939
4294
|
return error instanceof Error ? error : new Error(String(error));
|
|
@@ -3971,15 +4326,67 @@ function createPendingRestores(onFatalError) {
|
|
|
3971
4326
|
};
|
|
3972
4327
|
}
|
|
3973
4328
|
|
|
4329
|
+
// src/persistence/durability/snapshot-only/saved-size.ts
|
|
4330
|
+
function createSnapshotSavedSize(adapter, settleRestore) {
|
|
4331
|
+
const sizes = createStoredSizesRecord(false);
|
|
4332
|
+
const generations = /* @__PURE__ */ new Map();
|
|
4333
|
+
const store = adapterCheckpointStore(adapter);
|
|
4334
|
+
function replace(indexName, saved) {
|
|
4335
|
+
const generation = (generations.get(indexName) ?? 0) + 1;
|
|
4336
|
+
generations.set(indexName, generation);
|
|
4337
|
+
sizes.setSaved(indexName, saved);
|
|
4338
|
+
sizes.setMeasuring(indexName, saved === null);
|
|
4339
|
+
return generation;
|
|
4340
|
+
}
|
|
4341
|
+
async function measureAfterRestore(indexName, generation) {
|
|
4342
|
+
await settleRestore(indexName);
|
|
4343
|
+
const decoded = await readSegmentManifest(store, indexName);
|
|
4344
|
+
if (generations.get(indexName) !== generation) return;
|
|
4345
|
+
replace(indexName, decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED);
|
|
4346
|
+
}
|
|
4347
|
+
return {
|
|
4348
|
+
record(indexName, saved) {
|
|
4349
|
+
replace(indexName, saved);
|
|
4350
|
+
},
|
|
4351
|
+
hold: sizes.hold,
|
|
4352
|
+
forget(indexName) {
|
|
4353
|
+
generations.delete(indexName);
|
|
4354
|
+
sizes.forget(indexName);
|
|
4355
|
+
},
|
|
4356
|
+
drop(indexName) {
|
|
4357
|
+
generations.delete(indexName);
|
|
4358
|
+
sizes.drop(indexName);
|
|
4359
|
+
},
|
|
4360
|
+
access: {
|
|
4361
|
+
storedSizesOf: sizes.of,
|
|
4362
|
+
addStoredSizes(indexName, stats) {
|
|
4363
|
+
sizes.of(indexName).addTo(stats);
|
|
4364
|
+
},
|
|
4365
|
+
vectorSavedBytes(indexName) {
|
|
4366
|
+
return sizes.of(indexName).vectorSavedBytes;
|
|
4367
|
+
},
|
|
4368
|
+
async measureRestoredStorage(indexName, restored) {
|
|
4369
|
+
if (restored !== void 0) {
|
|
4370
|
+
replace(indexName, restored);
|
|
4371
|
+
return;
|
|
4372
|
+
}
|
|
4373
|
+
const generation = replace(indexName, null);
|
|
4374
|
+
void measureAfterRestore(indexName, generation).catch(() => void 0);
|
|
4375
|
+
}
|
|
4376
|
+
}
|
|
4377
|
+
};
|
|
4378
|
+
}
|
|
4379
|
+
|
|
3974
4380
|
// src/persistence/durability/snapshot-only/index.ts
|
|
3975
4381
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
3976
4382
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
3977
4383
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
3978
4384
|
const indexes = /* @__PURE__ */ new Map();
|
|
3979
|
-
const metadataWrites =
|
|
4385
|
+
const metadataWrites = createMetadataWriteQueue();
|
|
3980
4386
|
const loads = createIndexLoads(hooks.getVectorIndexes);
|
|
3981
4387
|
const restoreStorage = adapterRestoreStorage(adapter);
|
|
3982
4388
|
const restores = createPendingRestores(hooks.onFatalError);
|
|
4389
|
+
const savedSize = createSnapshotSavedSize(adapter, settleRestore);
|
|
3983
4390
|
let checkpointTimer = null;
|
|
3984
4391
|
let shuttingDown = false;
|
|
3985
4392
|
function settleRestore(indexName) {
|
|
@@ -3990,6 +4397,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3990
4397
|
if (state === void 0) {
|
|
3991
4398
|
state = createSnapshotIndexState();
|
|
3992
4399
|
indexes.set(indexName, state);
|
|
4400
|
+
hooks.linkStoredSizes(indexName, savedSize.hold(indexName));
|
|
3993
4401
|
}
|
|
3994
4402
|
return state;
|
|
3995
4403
|
}
|
|
@@ -4018,23 +4426,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4018
4426
|
const manager = hooks.getManager(indexName);
|
|
4019
4427
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
4020
4428
|
if (manager === void 0 || metadata === void 0) return;
|
|
4021
|
-
const {
|
|
4429
|
+
const { positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
|
|
4022
4430
|
adapter,
|
|
4023
4431
|
indexName,
|
|
4024
4432
|
metadata,
|
|
4025
4433
|
manager,
|
|
4026
4434
|
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
4027
4435
|
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
4028
|
-
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
|
|
4029
|
-
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
checkpointMetadata.partitionCount = partitionCount;
|
|
4035
|
-
const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
4036
|
-
await adapter.save(metadataKey(indexName), metadataBytes);
|
|
4037
|
-
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
|
|
4436
|
+
rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
|
|
4437
|
+
onCommitted: async (committed) => {
|
|
4438
|
+
savedSize.record(indexName, committed.saved);
|
|
4439
|
+
hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
|
|
4440
|
+
await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
|
|
4441
|
+
}
|
|
4038
4442
|
});
|
|
4039
4443
|
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
4040
4444
|
if (publisher !== void 0) {
|
|
@@ -4060,16 +4464,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4060
4464
|
);
|
|
4061
4465
|
}
|
|
4062
4466
|
function queueMetadataWrite(indexName, write) {
|
|
4063
|
-
|
|
4064
|
-
const queued = previous.then(() => settleRestore(indexName)).then(write);
|
|
4065
|
-
metadataWrites.set(
|
|
4066
|
-
indexName,
|
|
4067
|
-
queued.catch(() => void 0)
|
|
4068
|
-
);
|
|
4069
|
-
return queued;
|
|
4467
|
+
return metadataWrites.queue(indexName, () => settleRestore(indexName).then(write));
|
|
4070
4468
|
}
|
|
4071
|
-
async function saveMetadata(indexName, documentCount) {
|
|
4072
|
-
const metadata = hooks.buildMetadata(indexName, documentCount);
|
|
4469
|
+
async function saveMetadata(indexName, documentCount, savedBytes) {
|
|
4470
|
+
const metadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
|
|
4073
4471
|
if (metadata === void 0) return;
|
|
4074
4472
|
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
|
|
4075
4473
|
}
|
|
@@ -4079,9 +4477,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4079
4477
|
async function loadIndex(indexName, metadataOnly) {
|
|
4080
4478
|
await settleRestore(indexName);
|
|
4081
4479
|
const metaBytes = await adapter.load(metadataKey(indexName));
|
|
4082
|
-
if (metaBytes === null)
|
|
4083
|
-
return;
|
|
4084
|
-
}
|
|
4480
|
+
if (metaBytes === null) return;
|
|
4085
4481
|
const { metadata } = await readMetadataEnvelope(metaBytes);
|
|
4086
4482
|
const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
|
|
4087
4483
|
let derivedCount;
|
|
@@ -4120,6 +4516,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4120
4516
|
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
4121
4517
|
});
|
|
4122
4518
|
const indexState = getOrCreateIndexState(indexName);
|
|
4519
|
+
savedSize.record(indexName, recovered.saved);
|
|
4123
4520
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
4124
4521
|
const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
|
|
4125
4522
|
if (persistedSeqNo > 0) {
|
|
@@ -4128,7 +4525,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4128
4525
|
}
|
|
4129
4526
|
if (recovered.upgradedDocumentCount !== null || promoted) {
|
|
4130
4527
|
const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
|
|
4131
|
-
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(
|
|
4528
|
+
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount, recovered.saved.bytes)).catch(
|
|
4529
|
+
() => void 0
|
|
4530
|
+
);
|
|
4132
4531
|
}
|
|
4133
4532
|
}
|
|
4134
4533
|
async function recordMutation(record) {
|
|
@@ -4160,14 +4559,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4160
4559
|
try {
|
|
4161
4560
|
await indexState.applyChain;
|
|
4162
4561
|
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
4163
|
-
await metadataWrites.
|
|
4562
|
+
await metadataWrites.pending(indexName);
|
|
4164
4563
|
} catch (error) {
|
|
4165
4564
|
indexState.unloading = false;
|
|
4166
4565
|
throw error;
|
|
4167
4566
|
}
|
|
4168
4567
|
}
|
|
4169
4568
|
indexes.delete(indexName);
|
|
4170
|
-
metadataWrites.
|
|
4569
|
+
metadataWrites.forget(indexName);
|
|
4570
|
+
savedSize.forget(indexName);
|
|
4171
4571
|
}
|
|
4172
4572
|
return {
|
|
4173
4573
|
isActive() {
|
|
@@ -4249,6 +4649,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4249
4649
|
}
|
|
4250
4650
|
},
|
|
4251
4651
|
...queryHistoryAccess({ kind: "adapter", adapter }),
|
|
4652
|
+
...savedSize.access,
|
|
4252
4653
|
async removeIndex(indexName) {
|
|
4253
4654
|
await settleRestore(indexName).catch(() => void 0);
|
|
4254
4655
|
restores.forget(indexName);
|
|
@@ -4258,6 +4659,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4258
4659
|
await adapter.delete(key);
|
|
4259
4660
|
}
|
|
4260
4661
|
await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
|
|
4662
|
+
savedSize.drop(indexName);
|
|
4261
4663
|
},
|
|
4262
4664
|
async unloadIndex(indexName) {
|
|
4263
4665
|
await settleRestore(indexName);
|
|
@@ -4287,7 +4689,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4287
4689
|
}
|
|
4288
4690
|
|
|
4289
4691
|
// src/engine/durability-integration.ts
|
|
4290
|
-
function buildMetadata(indexName, hooks, documentCount) {
|
|
4692
|
+
function buildMetadata(indexName, hooks, documentCount, savedBytes) {
|
|
4291
4693
|
const config = hooks.getIndexConfig(indexName);
|
|
4292
4694
|
if (config === void 0) {
|
|
4293
4695
|
return void 0;
|
|
@@ -4297,7 +4699,8 @@ function buildMetadata(indexName, hooks, documentCount) {
|
|
|
4297
4699
|
indexName,
|
|
4298
4700
|
config,
|
|
4299
4701
|
manager === void 0 ? void 0 : metadataPartitionCount(manager),
|
|
4300
|
-
documentCount
|
|
4702
|
+
documentCount,
|
|
4703
|
+
savedBytes
|
|
4301
4704
|
);
|
|
4302
4705
|
}
|
|
4303
4706
|
function createDurabilityIntegration(tier, hooks) {
|
|
@@ -4305,7 +4708,7 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4305
4708
|
getManager: hooks.getManager,
|
|
4306
4709
|
getVectorFieldPaths: hooks.getVectorFieldPaths,
|
|
4307
4710
|
getVectorIndexes: hooks.getVectorIndexes,
|
|
4308
|
-
buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
|
|
4711
|
+
buildMetadata: (indexName, documentCount, savedBytes) => buildMetadata(indexName, hooks, documentCount, savedBytes),
|
|
4309
4712
|
buildTermsMetadata(indexName) {
|
|
4310
4713
|
const config = hooks.getTermsConfig(indexName);
|
|
4311
4714
|
return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
|
|
@@ -4320,7 +4723,8 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4320
4723
|
onCheckpointHeld: hooks.onCheckpointHeld,
|
|
4321
4724
|
onFatalError: hooks.onFatalError,
|
|
4322
4725
|
...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
|
|
4323
|
-
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
4726
|
+
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved },
|
|
4727
|
+
linkStoredSizes: hooks.linkStoredSizes
|
|
4324
4728
|
};
|
|
4325
4729
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
4326
4730
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
@@ -4574,7 +4978,7 @@ function createInvalidationIntegration(deps) {
|
|
|
4574
4978
|
|
|
4575
4979
|
// src/engine/durability-wiring.ts
|
|
4576
4980
|
function createDurabilityFromTier(tier, wiring) {
|
|
4577
|
-
|
|
4981
|
+
const integration = createDurabilityIntegration(tier, {
|
|
4578
4982
|
checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
|
|
4579
4983
|
getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
|
|
4580
4984
|
getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
|
|
@@ -4594,8 +4998,25 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4594
4998
|
},
|
|
4595
4999
|
queueDiskAdoption: wiring.queueDiskAdoption,
|
|
4596
5000
|
awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
|
|
4597
|
-
|
|
4598
|
-
|
|
5001
|
+
async createIndexFromMetadata(metadata, loadData) {
|
|
5002
|
+
await wiring.createIndexFromMetadata(metadata, loadData);
|
|
5003
|
+
const entry = wiring.indexRegistry.get(metadata.indexName);
|
|
5004
|
+
if (entry === void 0) return;
|
|
5005
|
+
entry.storedSizes = integration.manager.storedSizesOf(metadata.indexName);
|
|
5006
|
+
entry.storedSizes.recordSavedBytes(entry.savedBytes);
|
|
5007
|
+
},
|
|
5008
|
+
recordCheckpoint(indexName, documentCount, partitionCount, savedBytes) {
|
|
5009
|
+
const entry = wiring.indexRegistry.get(indexName);
|
|
5010
|
+
if (entry === void 0) return;
|
|
5011
|
+
entry.documentCount = documentCount;
|
|
5012
|
+
entry.partitionCount = partitionCount;
|
|
5013
|
+
entry.savedBytes = savedBytes;
|
|
5014
|
+
entry.storedSizes?.recordSavedBytes(savedBytes);
|
|
5015
|
+
},
|
|
5016
|
+
linkStoredSizes(indexName, sizes) {
|
|
5017
|
+
const entry = wiring.indexRegistry.get(indexName);
|
|
5018
|
+
if (entry !== void 0) entry.storedSizes = sizes;
|
|
5019
|
+
},
|
|
4599
5020
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
4600
5021
|
onCheckpointHeld: wiring.emitCheckpointHeld,
|
|
4601
5022
|
onFatalError: wiring.emitFatalError,
|
|
@@ -4609,6 +5030,7 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4609
5030
|
}
|
|
4610
5031
|
}
|
|
4611
5032
|
});
|
|
5033
|
+
return integration;
|
|
4612
5034
|
}
|
|
4613
5035
|
async function reloadIndexAndCopies(orchestrator, durability, indexName) {
|
|
4614
5036
|
const manager = durability?.manager;
|
|
@@ -5456,8 +5878,9 @@ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
|
5456
5878
|
partitionCount ?? void 0
|
|
5457
5879
|
);
|
|
5458
5880
|
}
|
|
5459
|
-
function restoredRegistryEntry(prepared, manager, deps) {
|
|
5881
|
+
function restoredRegistryEntry(indexName, prepared, manager, deps) {
|
|
5460
5882
|
return {
|
|
5883
|
+
storedSizes: deps.durability?.manager.storedSizesOf(indexName),
|
|
5461
5884
|
config: prepared.config,
|
|
5462
5885
|
language: prepared.language,
|
|
5463
5886
|
embeddingAdapter: prepared.embeddingAdapter,
|
|
@@ -5557,13 +5980,17 @@ async function commitToStorage(indexName, restored, previous, deps) {
|
|
|
5557
5980
|
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
5558
5981
|
await staging.commit();
|
|
5559
5982
|
} catch (error) {
|
|
5560
|
-
if (previousPartitions > 0)
|
|
5983
|
+
if (previousPartitions > 0) {
|
|
5984
|
+
durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
|
|
5985
|
+
await durable.measureRestoredStorage?.(indexName).catch(() => void 0);
|
|
5986
|
+
}
|
|
5561
5987
|
throw error;
|
|
5562
5988
|
}
|
|
5563
5989
|
for (const paged of pagedSegmentsOf(previous)) paged.close();
|
|
5564
5990
|
const restoredSegments = pagedSegmentsOf(restored.detached.manager);
|
|
5565
5991
|
for (const paged of restoredSegments) paged.suspend();
|
|
5566
5992
|
await staging.finish();
|
|
5993
|
+
await durable.measureRestoredStorage?.(indexName, restored.saved ?? void 0).catch(() => void 0);
|
|
5567
5994
|
const relocate = await staging.relocation().catch(() => null);
|
|
5568
5995
|
for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
|
|
5569
5996
|
if (relocate === null) return;
|
|
@@ -5572,6 +5999,7 @@ async function commitToStorage(indexName, restored, previous, deps) {
|
|
|
5572
5999
|
function activate(indexName, restored, entry, deps) {
|
|
5573
6000
|
deps.executor.installDetachedIndex(restored.detached);
|
|
5574
6001
|
deps.indexRegistry.set(indexName, entry);
|
|
6002
|
+
entry.storedSizes?.recordSavedBytes(entry.savedBytes);
|
|
5575
6003
|
restored.detached.attach();
|
|
5576
6004
|
deps.clearAnalysisStale(indexName);
|
|
5577
6005
|
const { prepared } = restored;
|
|
@@ -5611,6 +6039,7 @@ async function createFresh(indexName, restored, entry, deps) {
|
|
|
5611
6039
|
deps.copies.desyncIndex(indexName);
|
|
5612
6040
|
deps.executor.installDetachedIndex(restored.detached);
|
|
5613
6041
|
deps.indexRegistry.set(indexName, entry);
|
|
6042
|
+
entry.storedSizes?.recordSavedBytes(entry.savedBytes);
|
|
5614
6043
|
try {
|
|
5615
6044
|
await deps.indexState.registerReplacing(indexName, async () => {
|
|
5616
6045
|
await commitToStorage(indexName, restored, void 0, deps);
|
|
@@ -5629,9 +6058,10 @@ async function stageMetadata(indexName, staging, entry) {
|
|
|
5629
6058
|
await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
|
|
5630
6059
|
}
|
|
5631
6060
|
async function installRestoredIndex(indexName, restored, deps, history = null) {
|
|
5632
|
-
const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
|
|
6061
|
+
const entry = restoredRegistryEntry(indexName, restored.prepared, restored.detached.manager, deps);
|
|
5633
6062
|
const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
|
|
5634
6063
|
if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
|
|
6064
|
+
if (restored.saved !== null) entry.savedBytes = restored.saved.bytes;
|
|
5635
6065
|
let replaced;
|
|
5636
6066
|
try {
|
|
5637
6067
|
if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
|
|
@@ -5764,7 +6194,7 @@ async function loadStagedIndex(indexName, staged, staging, store, deps) {
|
|
|
5764
6194
|
deps.executor.discardDetachedIndex(detached);
|
|
5765
6195
|
throw error;
|
|
5766
6196
|
}
|
|
5767
|
-
return { prepared, detached, staging, checkpoint: manifest.checkpoint };
|
|
6197
|
+
return { prepared, detached, staging, checkpoint: manifest.checkpoint, saved: savedBytesOfManifest(manifest) };
|
|
5768
6198
|
}
|
|
5769
6199
|
async function restoreArchive(indexName, reader, deps) {
|
|
5770
6200
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
@@ -5865,7 +6295,7 @@ function insertSnapshotDocuments(envelope, manager) {
|
|
|
5865
6295
|
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
|
|
5866
6296
|
}
|
|
5867
6297
|
async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
5868
|
-
const entry = restoredRegistryEntry(prepared, built.manager, deps);
|
|
6298
|
+
const entry = restoredRegistryEntry(indexName, prepared, built.manager, deps);
|
|
5869
6299
|
const archive = await buildArchiveFromMemory({
|
|
5870
6300
|
indexName,
|
|
5871
6301
|
metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
|
|
@@ -5889,7 +6319,7 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
|
|
|
5889
6319
|
let manifest;
|
|
5890
6320
|
try {
|
|
5891
6321
|
insertSnapshotDocuments(envelope, built.manager);
|
|
5892
|
-
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
|
|
6322
|
+
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [], saved: null };
|
|
5893
6323
|
manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
|
|
5894
6324
|
} catch (err) {
|
|
5895
6325
|
deps.executor.discardDetachedIndex(built);
|