@delali/narsil 0.3.1-next.180 → 0.3.1-next.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
  4. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
  5. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  6. package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
  7. package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
  8. package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
  9. package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
  10. package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
  11. package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
  12. package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
  13. package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
  14. package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
  15. package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
  16. package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
  17. package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
  18. package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
  19. package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
  20. package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
  21. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  22. package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
  23. package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
  24. package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
  25. package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
  26. package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
  27. package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
  28. package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
  29. package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
  30. package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
  31. package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
  32. package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
  33. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  34. package/dist/client.d.ts +6 -6
  35. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  36. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  37. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
  38. package/dist/distribution.d.ts +4 -4
  39. package/dist/distribution.mjs +391 -138
  40. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  41. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  42. package/dist/index.browser.mjs +1 -1
  43. package/dist/index.d.ts +4 -4
  44. package/dist/index.mjs +1 -1
  45. package/dist/index.react-native.mjs +1 -1
  46. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
  47. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
  48. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
  49. package/dist/partial-words.browser.mjs +1 -1
  50. package/dist/partial-words.mjs +1 -1
  51. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  52. package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
  53. package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
  54. package/dist/query-suggestions/merge-worker.mjs +4 -4
  55. package/dist/query-suggestions.browser.mjs +1 -1
  56. package/dist/query-suggestions.d.ts +1 -1
  57. package/dist/query-suggestions.mjs +2 -2
  58. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
  59. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
  60. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
  61. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
  62. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
  63. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
  64. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  65. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
  66. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
  67. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
  68. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
  69. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
  70. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
  71. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
  72. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
  73. package/dist/react-native/partial-words.mjs +1 -1
  74. package/dist/react-native/query-suggestions.mjs +1 -1
  75. package/dist/react-native/sync.mjs +1 -1
  76. package/dist/react-native/synonyms.mjs +1 -1
  77. package/dist/react.d.ts +7 -7
  78. package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
  79. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
  80. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
  81. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
  82. package/dist/server.d.ts +8 -8
  83. package/dist/server.mjs +16 -16
  84. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
  85. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
  86. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
  87. package/dist/sync/filesystem.d.ts +2 -2
  88. package/dist/sync/filesystem.mjs +6 -6
  89. package/dist/sync/indexeddb.d.ts +2 -2
  90. package/dist/sync.browser.mjs +1 -1
  91. package/dist/sync.d.ts +3 -3
  92. package/dist/sync.mjs +1 -1
  93. package/dist/synonyms.browser.mjs +1 -1
  94. package/dist/synonyms.mjs +1 -1
  95. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  96. package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
  97. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  98. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
  99. package/dist/workers/entry.mjs +7 -7
  100. package/openapi.json +32 -0
  101. package/package.json +7 -7
@@ -1,34 +1,34 @@
1
- import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES, scheduleSyncRetention } from './chunk-XITVDRRH.mjs';
2
- export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-XITVDRRH.mjs';
3
- import { openPagedSegmentScan, countsAsLatestHolder, buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, chunksOf, isRecord, nestedSchemaOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, bytesThatSegmentsHoldInMemory, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, requireSavedPartitionCount, createDetachedIndex, checkPagedSegmentFully, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readMetadataEnvelope, snapshotCheckpointFor, whileGraphsStayComplete, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, readSegmentManifest, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, highestPartitionIdOf, growPartitionsTo, createMarkerWriter, metadataConfigOf, durableMetadataConfigOf } from './chunk-7FC2ZILY.mjs';
4
- export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedSegmentFully, countsAsLatestHolder, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, openPagedSegmentScan, releaseHeldAnalysisChange, requireSavedPartitionCount, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-7FC2ZILY.mjs';
5
- import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-YPV2L7M5.mjs';
6
- export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-YPV2L7M5.mjs';
7
- import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-LOQHA6QK.mjs';
8
- export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-LOQHA6QK.mjs';
1
+ import { scheduleSyncRetention, createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-M3H7CTIC.mjs';
2
+ export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-M3H7CTIC.mjs';
3
+ import { buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, metadataConfigOf, durableMetadataConfigOf, chunksOf, isRecord, nestedSchemaOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, requireSavedPartitionCount, createDetachedIndex, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readSegmentManifest, readMetadataEnvelope, snapshotCheckpointFor, whileGraphsStayComplete, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, highestPartitionIdOf, growPartitionsTo, createMarkerWriter } from './chunk-ZVZ2V2PZ.mjs';
4
+ export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, requireSavedPartitionCount, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-ZVZ2V2PZ.mjs';
5
+ import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-CKS4QQEC.mjs';
6
+ export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-CKS4QQEC.mjs';
7
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-SA7U67XF.mjs';
8
+ export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-SA7U67XF.mjs';
9
9
  import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-X4UMDSB4.mjs';
10
- import { readSnapshotArchive, decodeSegmentManifest, isRecord as isRecord$1, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-UISV46WX.mjs';
11
- export { readSnapshotArchive } from './chunk-UISV46WX.mjs';
12
- import { createDurableDirectory } from './chunk-34IOOYXA.mjs';
13
- import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, readSegmentFile, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-F3FDHXCJ.mjs';
14
- export { checkPagedContents, checkPagedSelections, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-F3FDHXCJ.mjs';
15
- import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-ZB5X5WAK.mjs';
10
+ import { readSnapshotArchive, decodeSegmentManifest, isRecord as isRecord$1, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-NPXORGMK.mjs';
11
+ export { readSnapshotArchive } from './chunk-NPXORGMK.mjs';
12
+ import { createDurableDirectory } from './chunk-6ICOWB6I.mjs';
13
+ import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, openPagedSegmentScan, countsAsLatestHolder, writePagedSegment, bytesThatSegmentsHoldInMemory, savedBytesOfManifest, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, checkPagedSegmentFully, readSegmentFile, NOTHING_SAVED, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, checkpointGarbage, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-3OEXOSWI.mjs';
14
+ export { bytesThatSegmentsHoldInMemory, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSegmentScan, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-3OEXOSWI.mjs';
15
+ import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-TTTBE4TB.mjs';
16
16
  import './chunk-5FLRXF73.mjs';
17
17
  import './chunk-JDDXV6UF.mjs';
18
- import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-KHJPMZE2.mjs';
18
+ import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-PK3BGGHO.mjs';
19
19
  import { getFs } from './chunk-HLPRHOSO.mjs';
20
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
20
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-MSIN4NM3.mjs';
21
21
  import { encode } from './chunk-CV3Q2EKO.mjs';
22
22
  export { bindStorageHost } from './chunk-CV3Q2EKO.mjs';
23
23
  import { sha256 } from './chunk-LNGE7SLY.mjs';
24
24
  import { setNestedValue } from './chunk-I2WJCRIH.mjs';
25
- import { rewriteSpillOf } from './chunk-XFOL77WH.mjs';
26
- import { deserializePartition } from './chunk-R7PLSVK3.mjs';
25
+ import { rewriteSpillOf } from './chunk-YWSIRKKQ.mjs';
26
+ import { deserializePartition } from './chunk-OQANB7SJ.mjs';
27
27
  import { decodeVectorIndexParts, decodeVectorFilePayload, isDead, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, graphPartsOfEarlierPayload, decodeVectorGraphPayload } from './chunk-TNEU32EN.mjs';
28
28
  export { createHNSWIndex, osqBitsOf } from './chunk-TNEU32EN.mjs';
29
29
  import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
30
- import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
31
- import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createPartitionRouter, buildSegmentPayloads, createRequestId } from './chunk-2C3BXZGA.mjs';
30
+ import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
31
+ import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createPartitionRouter, buildSegmentPayloads, createRequestId } from './chunk-CWXR7OP4.mjs';
32
32
  import { encodeDocument } from './chunk-KLVMK3R5.mjs';
33
33
  import './chunk-LPRRFCVU.mjs';
34
34
  import { decode } from './chunk-GXCDWLQI.mjs';
@@ -49,28 +49,55 @@ import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
49
49
  function yieldToEventLoop() {
50
50
  return new Promise((resolve) => setTimeout(resolve, 0));
51
51
  }
52
+ function newerTombstonesOf(prepared, at) {
53
+ const docIds = /* @__PURE__ */ new Set();
54
+ for (let newer = at + 1; newer < prepared.length; newer++) {
55
+ const access = prepared[newer].segment.paged;
56
+ if (access === null) continue;
57
+ for (const docId of access.snapshot.tombstones) docIds.add(docId);
58
+ }
59
+ return docIds;
60
+ }
61
+ function scansEveryId(segment, tombstoned) {
62
+ const held = segment.arrays.idTable.count;
63
+ return tombstoned.size * Math.ceil(Math.log2(held + 1)) > held;
64
+ }
65
+ function tombstoneScannedOrdinal(segment, tombstoned, ordinal) {
66
+ const docId = segment.arrays.idTable.idAt(ordinal);
67
+ if (tombstoned.has(docId)) segment.prepareTombstone(docId, void 0, void 0, ordinal)?.publish();
68
+ }
52
69
  function applyNewerTombstones(prepared) {
53
70
  for (let at = 0; at < prepared.length; at++) {
54
71
  if (!prepared[at].isNew) continue;
55
- for (let newer = at + 1; newer < prepared.length; newer++) {
56
- const access = prepared[newer].segment.paged;
57
- if (access === null) continue;
58
- for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
72
+ const { segment } = prepared[at];
73
+ const tombstoned = newerTombstonesOf(prepared, at);
74
+ if (!scansEveryId(segment, tombstoned)) {
75
+ for (const docId of tombstoned) segment.tombstoneDocument(docId);
76
+ continue;
59
77
  }
78
+ const held = segment.arrays.idTable.count;
79
+ for (let ordinal = 0; ordinal < held; ordinal++) tombstoneScannedOrdinal(segment, tombstoned, ordinal);
60
80
  }
61
81
  }
62
82
  async function applyNewerTombstonesInTurns(prepared) {
63
- let applied = 0;
83
+ let steps = 0;
64
84
  for (let at = 0; at < prepared.length; at++) {
65
85
  if (!prepared[at].isNew) continue;
66
- for (let newer = at + 1; newer < prepared.length; newer++) {
67
- const access = prepared[newer].segment.paged;
68
- if (access === null) continue;
69
- for (const docId of access.snapshot.tombstones) {
70
- prepared[at].segment.tombstoneDocument(docId);
71
- applied += 1;
72
- if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
86
+ const { segment } = prepared[at];
87
+ const tombstoned = newerTombstonesOf(prepared, at);
88
+ if (!scansEveryId(segment, tombstoned)) {
89
+ for (const docId of tombstoned) {
90
+ segment.tombstoneDocument(docId);
91
+ steps += 1;
92
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
73
93
  }
94
+ continue;
95
+ }
96
+ const held = segment.arrays.idTable.count;
97
+ for (let ordinal = 0; ordinal < held; ordinal++) {
98
+ tombstoneScannedOrdinal(segment, tombstoned, ordinal);
99
+ steps += 1;
100
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
74
101
  }
75
102
  }
76
103
  }
@@ -439,6 +466,9 @@ function wholePartitionsWhereMostChanged(manager, rules) {
439
466
  return whole;
440
467
  };
441
468
  }
469
+ function liveDocumentsOf(manager, targets) {
470
+ return targets.map(({ partitionId }) => ({ partitionId, documentCount: manager.getPartition(partitionId).count() }));
471
+ }
442
472
  async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
443
473
  const whileNoBuildPassOrMutationRuns = (capture2) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture2));
444
474
  return whileNoBuildPassOrMutationRuns(
@@ -481,7 +511,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
481
511
  vectorPlans,
482
512
  wholePartitions,
483
513
  recordsLeftInLog,
484
- unreplayedCutoffs: cutoffs
514
+ unreplayedCutoffs: cutoffs,
515
+ liveDocuments: memoryMatchesTheTargets && cutoffs.size === 0 ? liveDocumentsOf(manager, targets) : null
485
516
  };
486
517
  } catch (error) {
487
518
  for (const plan of vectorPlans.values()) plan.release();
@@ -702,7 +733,8 @@ function workerCanWriteEverySegment(input) {
702
733
  }
703
734
  async function writeIndexCheckpointSegments(input) {
704
735
  const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
705
- const { keepsWriteRecords } = input;
736
+ const { keepsWriteRecords, liveDocuments, rewriteBytesPerCheckpoint } = input;
737
+ const counted = liveDocuments === null ? {} : { liveDocuments, rewriteBytesPerCheckpoint };
706
738
  const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
707
739
  root: directory.root,
708
740
  metadata,
@@ -710,7 +742,8 @@ async function writeIndexCheckpointSegments(input) {
710
742
  compactionThreshold,
711
743
  prior,
712
744
  capturedPartitions: [...capturedPartitions],
713
- keepsWriteRecords
745
+ keepsWriteRecords,
746
+ ...counted
714
747
  }) : null;
715
748
  if (offloaded !== null) {
716
749
  return offloaded;
@@ -724,7 +757,8 @@ async function writeIndexCheckpointSegments(input) {
724
757
  prior,
725
758
  wholePartitions,
726
759
  capturedPartitions,
727
- keepsWriteRecords
760
+ keepsWriteRecords,
761
+ ...counted
728
762
  });
729
763
  }
730
764
 
@@ -921,23 +955,28 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
921
955
  metadata,
922
956
  targets,
923
957
  compactionThreshold: input.compactionThreshold,
958
+ rewriteBytesPerCheckpoint: input.rewriteBytesPerCheckpoint,
924
959
  canOffload: input.canOffload,
925
960
  prior: priorForLayout(priorManifest, capture2.partitionCount),
926
961
  wholePartitions: capture2.wholePartitions.contents,
927
962
  capturedPartitions: capture2.wholePartitions.captured,
928
- keepsWriteRecords: writeRecords !== null
963
+ keepsWriteRecords: writeRecords !== null,
964
+ liveDocuments: capture2.liveDocuments
929
965
  })
930
966
  );
931
967
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
932
968
  if (aTargetFailed(input.indexState, targets)) return 0;
933
969
  const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
970
+ input.recordSavedBytes(written.saved);
934
971
  input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
935
972
  recordWrittenVectors(vectorIndexes, liveVectors.written);
936
973
  await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
937
974
  const changes = changesSinceCapture();
938
975
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
976
+ await writeCheckpointMetadata(input, documentCount, capture2.partitionCount, written.saved.bytes);
939
977
  if (!input.holdsCheckpointFiles()) {
940
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
978
+ const garbage = await checkpointGarbage(store, input.indexName, written.manifest);
979
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, garbage);
941
980
  const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
942
981
  const keptForSync = await input.keepsCheckpointKey();
943
982
  await removeCheckpointGarbage(
@@ -945,19 +984,6 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
945
984
  removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
946
985
  );
947
986
  }
948
- const checkpointDocumentCount = documentCount;
949
- await input.queueMetadataWrite(input.indexName, async () => {
950
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
951
- if (checkpointMetadata === void 0) {
952
- return;
953
- }
954
- const bytes = await writeMetadataEnvelope(
955
- { ...checkpointMetadata, partitionCount: capture2.partitionCount },
956
- { checksum: true }
957
- );
958
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
959
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture2.partitionCount);
960
- });
961
987
  const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
962
988
  await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
963
989
  if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
@@ -975,6 +1001,18 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
975
1001
  const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
976
1002
  return anotherIsDue ? eligibleBeyond : capture2.recordsLeftInLog;
977
1003
  }
1004
+ async function writeCheckpointMetadata(input, documentCount, partitionCount, savedBytes) {
1005
+ const { indexName, hooks } = input;
1006
+ await input.queueMetadataWrite(indexName, async () => {
1007
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
1008
+ if (checkpointMetadata === void 0) {
1009
+ return;
1010
+ }
1011
+ const bytes = await writeMetadataEnvelope({ ...checkpointMetadata, partitionCount }, { checksum: true });
1012
+ await input.directory.atomicWrite(`${indexName}/meta`, bytes);
1013
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, savedBytes);
1014
+ });
1015
+ }
978
1016
  function recordsBeyond(indexState, targets, cutoffs) {
979
1017
  let records = 0;
980
1018
  for (const [partitionId, partition] of indexState.partitions) {
@@ -1051,6 +1089,258 @@ function createHeldCheckpoints(hooks) {
1051
1089
  };
1052
1090
  }
1053
1091
 
1092
+ // src/persistence/durability/index-stored-sizes.ts
1093
+ function refreshStoredSizes(sizes) {
1094
+ const known = sizes.saved !== null || !sizes.measuring && sizes.held;
1095
+ sizes.known = known;
1096
+ if (known) {
1097
+ sizes.savedBytes = sizes.saved?.bytes ?? 0;
1098
+ sizes.vectorSavedBytes = sizes.saved?.vectorBytes ?? 0;
1099
+ sizes.logBytes = sizes.tracksLog ? sizes.logged?.bytes ?? 0 : void 0;
1100
+ return;
1101
+ }
1102
+ sizes.savedBytes = sizes.measuring ? void 0 : sizes.recorded;
1103
+ sizes.vectorSavedBytes = void 0;
1104
+ sizes.logBytes = void 0;
1105
+ }
1106
+ function savedBytesByPartitionOf(saved) {
1107
+ const byPartition = [];
1108
+ if (saved === null) return byPartition;
1109
+ for (const { partitionId, bytes } of saved.partitions) byPartition[partitionId] = bytes;
1110
+ return byPartition;
1111
+ }
1112
+ function createTrackedSizes(tracksLog) {
1113
+ const sizes = {
1114
+ savedBytes: void 0,
1115
+ logBytes: void 0,
1116
+ vectorSavedBytes: void 0,
1117
+ saved: null,
1118
+ savedBytesByPartition: [],
1119
+ measuring: false,
1120
+ held: false,
1121
+ recorded: void 0,
1122
+ logged: null,
1123
+ known: false,
1124
+ tracksLog,
1125
+ addTo(stats) {
1126
+ if (!sizes.known) return;
1127
+ const saved = sizes.savedBytesByPartition;
1128
+ const logged = sizes.tracksLog ? sizes.logged?.bytesByPartition ?? [] : null;
1129
+ for (const partition of stats) {
1130
+ partition.savedBytes = saved[partition.partitionId] ?? 0;
1131
+ if (logged !== null) partition.logBytes = logged[partition.partitionId] ?? 0;
1132
+ }
1133
+ },
1134
+ recordSavedBytes(bytes) {
1135
+ sizes.recorded = bytes;
1136
+ refreshStoredSizes(sizes);
1137
+ }
1138
+ };
1139
+ return sizes;
1140
+ }
1141
+ function createStoredSizesRecord(tracksLog) {
1142
+ const sizesByIndex = /* @__PURE__ */ new Map();
1143
+ function of(indexName) {
1144
+ let sizes = sizesByIndex.get(indexName);
1145
+ if (sizes === void 0) {
1146
+ sizes = createTrackedSizes(tracksLog);
1147
+ sizesByIndex.set(indexName, sizes);
1148
+ }
1149
+ return sizes;
1150
+ }
1151
+ function forget(indexName) {
1152
+ const sizes = sizesByIndex.get(indexName);
1153
+ if (sizes === void 0) return;
1154
+ sizes.saved = null;
1155
+ sizes.savedBytesByPartition = [];
1156
+ sizes.measuring = false;
1157
+ sizes.held = false;
1158
+ refreshStoredSizes(sizes);
1159
+ }
1160
+ return {
1161
+ of,
1162
+ setSaved(indexName, saved) {
1163
+ const sizes = of(indexName);
1164
+ sizes.saved = saved;
1165
+ sizes.savedBytesByPartition = savedBytesByPartitionOf(saved);
1166
+ refreshStoredSizes(sizes);
1167
+ },
1168
+ setMeasuring(indexName, measuring) {
1169
+ const sizes = of(indexName);
1170
+ sizes.measuring = measuring;
1171
+ refreshStoredSizes(sizes);
1172
+ },
1173
+ hold(indexName) {
1174
+ const sizes = of(indexName);
1175
+ if (!sizes.held) {
1176
+ sizes.held = true;
1177
+ refreshStoredSizes(sizes);
1178
+ }
1179
+ return sizes;
1180
+ },
1181
+ forget,
1182
+ drop(indexName) {
1183
+ forget(indexName);
1184
+ sizesByIndex.delete(indexName);
1185
+ }
1186
+ };
1187
+ }
1188
+
1189
+ // src/persistence/durability/log-bytes.ts
1190
+ var LOG_SEGMENT_KEY = /^(.+)\/wal\/(\d+)\/\d{16}$/;
1191
+ var HISTORY_SEGMENT_KEY = /^(.+)\/history\/(\d+)\/\d{16}-\d{16}$/;
1192
+ function loggedFileOf(key) {
1193
+ const match = LOG_SEGMENT_KEY.exec(key) ?? HISTORY_SEGMENT_KEY.exec(key);
1194
+ if (match === null) return null;
1195
+ const partitionId = Number.parseInt(match[2], 10);
1196
+ return Number.isSafeInteger(partitionId) ? { indexName: match[1], partitionId } : null;
1197
+ }
1198
+ function emptyLog() {
1199
+ return { files: /* @__PURE__ */ new Map(), bytes: 0, bytesByPartition: [], sizes: null };
1200
+ }
1201
+ function addBytes(log, partitionId, delta) {
1202
+ log.bytes += delta;
1203
+ log.bytesByPartition[partitionId] = (log.bytesByPartition[partitionId] ?? 0) + delta;
1204
+ if (log.sizes !== null) refreshStoredSizes(log.sizes);
1205
+ }
1206
+ function makeCurrent(previous, log, sizes) {
1207
+ if (previous !== void 0 && previous !== log) previous.sizes = null;
1208
+ log.sizes = sizes;
1209
+ sizes.logged = log;
1210
+ refreshStoredSizes(sizes);
1211
+ }
1212
+ function setFileBytes(log, key, partitionId, bytes) {
1213
+ const previous = log.files.get(key);
1214
+ if (previous !== void 0) addBytes(log, previous.partitionId, -previous.bytes);
1215
+ log.files.set(key, { partitionId, bytes });
1216
+ addBytes(log, partitionId, bytes);
1217
+ }
1218
+ function removeFile(log, key) {
1219
+ const previous = log.files.get(key);
1220
+ if (previous === void 0) return void 0;
1221
+ log.files.delete(key);
1222
+ addBytes(log, previous.partitionId, -previous.bytes);
1223
+ return previous;
1224
+ }
1225
+ function createLogBytesLedger(base, sizes) {
1226
+ const logs = /* @__PURE__ */ new Map();
1227
+ function logOf(indexName) {
1228
+ let log = logs.get(indexName);
1229
+ if (log === void 0) {
1230
+ log = emptyLog();
1231
+ logs.set(indexName, log);
1232
+ makeCurrent(void 0, log, sizes.of(indexName));
1233
+ }
1234
+ return log;
1235
+ }
1236
+ function track(key, bytes) {
1237
+ const file = loggedFileOf(key);
1238
+ if (file !== null) setFileBytes(logOf(file.indexName), key, file.partitionId, bytes);
1239
+ }
1240
+ function untrack(key) {
1241
+ const file = loggedFileOf(key);
1242
+ if (file === null) return void 0;
1243
+ const log = logs.get(file.indexName);
1244
+ return log === void 0 ? void 0 : removeFile(log, key);
1245
+ }
1246
+ async function currentLength(key) {
1247
+ return await base.byteLength?.(key) ?? (await base.read(key))?.byteLength ?? 0;
1248
+ }
1249
+ function trackedHandle(key, file, handle) {
1250
+ async function remeasure() {
1251
+ const log = logs.get(file.indexName);
1252
+ if (log?.files.has(key) === true) setFileBytes(log, key, file.partitionId, await handle.size());
1253
+ }
1254
+ return {
1255
+ async append(bytes) {
1256
+ try {
1257
+ await handle.append(bytes);
1258
+ } catch (error) {
1259
+ await remeasure().catch(() => void 0);
1260
+ throw error;
1261
+ }
1262
+ const log = logs.get(file.indexName);
1263
+ const entry = log?.files.get(key);
1264
+ if (log === void 0 || entry === void 0) return;
1265
+ entry.bytes += bytes.byteLength;
1266
+ addBytes(log, file.partitionId, bytes.byteLength);
1267
+ },
1268
+ sync: () => handle.sync(),
1269
+ async truncate(byteLength2) {
1270
+ try {
1271
+ await handle.truncate(byteLength2);
1272
+ } finally {
1273
+ await remeasure().catch(() => void 0);
1274
+ }
1275
+ },
1276
+ size: () => handle.size(),
1277
+ close: () => handle.close()
1278
+ };
1279
+ }
1280
+ const directory = {
1281
+ root: base.root,
1282
+ async appendHandle(key) {
1283
+ const handle = await base.appendHandle(key);
1284
+ const file = loggedFileOf(key);
1285
+ if (file === null) return handle;
1286
+ if (logs.get(file.indexName)?.files.has(key) !== true) track(key, await handle.size());
1287
+ return trackedHandle(key, file, handle);
1288
+ },
1289
+ markerHandle: (key) => base.markerHandle(key),
1290
+ async atomicWrite(key, data) {
1291
+ await base.atomicWrite(key, data);
1292
+ if (loggedFileOf(key) === null) return;
1293
+ let bytes = 0;
1294
+ for (const chunk of Array.isArray(data) ? data : [data]) bytes += chunk.byteLength;
1295
+ track(key, bytes);
1296
+ },
1297
+ syncDirectoryOf: (key) => base.syncDirectoryOf(key),
1298
+ read: (key) => base.read(key),
1299
+ async remove(key) {
1300
+ await base.remove(key);
1301
+ untrack(key);
1302
+ },
1303
+ async move(from, to) {
1304
+ await base.move(from, to);
1305
+ const moved = untrack(from);
1306
+ if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
1307
+ },
1308
+ list: (prefix) => base.list(prefix),
1309
+ pathOf: (key) => base.pathOf(key)
1310
+ };
1311
+ const { readRange, byteLength, identity, lock } = base;
1312
+ if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
1313
+ if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
1314
+ if (identity !== void 0) directory.identity = () => identity.call(base);
1315
+ if (lock !== void 0) directory.lock = () => lock.call(base);
1316
+ return {
1317
+ directory,
1318
+ async seed(indexName) {
1319
+ const log = emptyLog();
1320
+ for (const prefix of [`${indexName}/wal/`, `${indexName}/history/`]) {
1321
+ for (const key of await base.list(prefix)) {
1322
+ const file = loggedFileOf(key);
1323
+ if (file === null || file.indexName !== indexName) continue;
1324
+ setFileBytes(log, key, file.partitionId, await currentLength(key));
1325
+ }
1326
+ }
1327
+ const previous = logs.get(indexName);
1328
+ logs.set(indexName, log);
1329
+ makeCurrent(previous, log, sizes.of(indexName));
1330
+ },
1331
+ forget(indexName) {
1332
+ const log = logs.get(indexName);
1333
+ if (log === void 0) return;
1334
+ logs.delete(indexName);
1335
+ if (log.sizes !== null) {
1336
+ log.sizes.logged = null;
1337
+ refreshStoredSizes(log.sizes);
1338
+ log.sizes = null;
1339
+ }
1340
+ }
1341
+ };
1342
+ }
1343
+
1054
1344
  // src/persistence/durability/restore-journal.ts
1055
1345
  var RESTORE_JOURNAL_PREFIX = ".swap-";
1056
1346
  var RESTORE_STAGING_PREFIX = ".restore-";
@@ -1447,6 +1737,7 @@ function createIndexUnloading(deps) {
1447
1737
  }
1448
1738
  await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
1449
1739
  deps.forgetSyncState(indexName);
1740
+ deps.dropStoredBytes(indexName);
1450
1741
  },
1451
1742
  async unloadIndex(indexName) {
1452
1743
  const indexState = indexes.get(indexName);
@@ -1457,6 +1748,7 @@ function createIndexUnloading(deps) {
1457
1748
  indexes.delete(indexName);
1458
1749
  metadataWrites.forget(indexName);
1459
1750
  deps.forgetSyncState(indexName);
1751
+ deps.forgetStoredBytes(indexName);
1460
1752
  }
1461
1753
  };
1462
1754
  }
@@ -1920,7 +2212,7 @@ function createSyncStorage(deps) {
1920
2212
 
1921
2213
  // src/persistence/durability/manager-storage.ts
1922
2214
  function createIndexStorageAccess(deps) {
1923
- const { directory, indexes } = deps;
2215
+ const { directory, logBytes, sizes } = deps;
1924
2216
  const archiveAccess = createArchiveCheckpointAccess({
1925
2217
  directory,
1926
2218
  held: deps.heldCheckpointFiles,
@@ -1932,7 +2224,7 @@ function createIndexStorageAccess(deps) {
1932
2224
  directory,
1933
2225
  openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
1934
2226
  inCheckpointTurn: (indexName, work) => {
1935
- const indexState = indexes.get(indexName);
2227
+ const indexState = deps.indexes.get(indexName);
1936
2228
  if (indexState === void 0) return work();
1937
2229
  return checkpointInTurn(indexState, {
1938
2230
  joinsInFlight: false,
@@ -1945,12 +2237,54 @@ function createIndexStorageAccess(deps) {
1945
2237
  const unloading = createIndexUnloading({
1946
2238
  directory,
1947
2239
  hooks: deps.hooks,
1948
- indexes,
2240
+ indexes: deps.indexes,
1949
2241
  metadataWrites: deps.metadataWrites,
1950
2242
  closeWriterReportingFailure: deps.closeWriterReportingFailure,
1951
- forgetSyncState: syncStorage.forget
2243
+ forgetSyncState: syncStorage.forget,
2244
+ forgetStoredBytes: (indexName) => {
2245
+ sizes.forget(indexName);
2246
+ logBytes.forget(indexName);
2247
+ },
2248
+ dropStoredBytes: (indexName) => {
2249
+ sizes.drop(indexName);
2250
+ logBytes.forget(indexName);
2251
+ }
1952
2252
  });
1953
- return { archiveAccess, syncStorage, unloading };
2253
+ async function measureStoredBytes(indexName, saved) {
2254
+ sizes.setSaved(indexName, saved);
2255
+ await logBytes.seed(indexName);
2256
+ }
2257
+ const recoveryDeps = {
2258
+ directory,
2259
+ hooks: deps.hooks,
2260
+ compactionThreshold: deps.compactionThreshold,
2261
+ registerPartition: deps.registerPartition,
2262
+ holdsCheckpointFiles: (indexName) => deps.heldCheckpointFiles.has(indexName),
2263
+ queueMetadataWrite: deps.metadataWrites.queue,
2264
+ syncStorage,
2265
+ measureStoredBytes
2266
+ };
2267
+ const storedBytesAccess = {
2268
+ storedSizesOf: sizes.of,
2269
+ addStoredSizes(indexName, stats) {
2270
+ sizes.of(indexName).addTo(stats);
2271
+ },
2272
+ vectorSavedBytes(indexName) {
2273
+ return sizes.of(indexName).vectorSavedBytes;
2274
+ },
2275
+ async measureRestoredStorage(indexName, restored) {
2276
+ if (restored !== void 0) {
2277
+ await measureStoredBytes(indexName, restored);
2278
+ return;
2279
+ }
2280
+ const decoded = await readSegmentManifest(directoryCheckpointStore(directory), indexName);
2281
+ await measureStoredBytes(
2282
+ indexName,
2283
+ decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED
2284
+ );
2285
+ }
2286
+ };
2287
+ return { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps };
1954
2288
  }
1955
2289
 
1956
2290
  // src/persistence/durability/metadata-write-queue.ts
@@ -2476,7 +2810,7 @@ function createWalWriter(directory, config) {
2476
2810
  }
2477
2811
 
2478
2812
  // src/persistence/durability/partition-registry.ts
2479
- function createPartitionRegistry(directory, segmentMaxBytes) {
2813
+ function createPartitionRegistry(directory, segmentMaxBytes, onIndexHeld) {
2480
2814
  const indexes = /* @__PURE__ */ new Map();
2481
2815
  function indexState(indexName) {
2482
2816
  let state = indexes.get(indexName);
@@ -2490,6 +2824,7 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
2490
2824
  stalledWrites: []
2491
2825
  };
2492
2826
  indexes.set(indexName, state);
2827
+ onIndexHeld(indexName);
2493
2828
  }
2494
2829
  return state;
2495
2830
  }
@@ -3267,19 +3602,18 @@ async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
3267
3602
  const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
3268
3603
  recordWrittenVectors(deps.vectorIndexes, vectors.written);
3269
3604
  await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
3270
- await removeCheckpointGarbage(
3271
- deps.store,
3272
- await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, committed.garbage)
3273
- );
3605
+ const garbage = await checkpointGarbage(deps.store, deps.indexName, committed.manifest);
3606
+ await removeCheckpointGarbage(deps.store, await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, garbage));
3607
+ return committed.saved;
3274
3608
  }
3275
3609
  async function upgradeEarlierLayout(deps, earlier) {
3276
3610
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
3277
3611
  try {
3278
3612
  const loaded = await loadEarlier(deps, earlier);
3279
3613
  const targets = await deps.replay(loaded.checkpoint);
3280
- await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
3614
+ const saved = await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
3281
3615
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
3282
- return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
3616
+ return { checkpoint: targets, documentCount: deps.manager.countDocuments(), saved };
3283
3617
  } catch (error) {
3284
3618
  const failure = error instanceof Error ? error : new Error(String(error));
3285
3619
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
@@ -3299,11 +3633,11 @@ async function recoverCheckpoint(deps) {
3299
3633
  const earlier = await earlierLayoutOf(deps, decoded);
3300
3634
  if (earlier !== null) {
3301
3635
  const upgraded = await upgradeEarlierLayout(deps, earlier);
3302
- return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
3636
+ return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount, saved: upgraded.saved };
3303
3637
  }
3304
3638
  if (decoded === null || decoded.kind !== "current") {
3305
3639
  if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
3306
- return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
3640
+ return { checkpoint: await deps.replay([]), upgradedDocumentCount: null, saved: NOTHING_SAVED };
3307
3641
  }
3308
3642
  await deps.store.remove(snapshotBundleKey(deps.indexName));
3309
3643
  if (!deps.keepsUnlistedFiles) {
@@ -3316,7 +3650,11 @@ async function recoverCheckpoint(deps) {
3316
3650
  vectorIndexes: deps.vectorIndexes,
3317
3651
  writeRecords: deps.writeRecords ?? null
3318
3652
  });
3319
- return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
3653
+ return {
3654
+ checkpoint: await deps.replay(checkpoint),
3655
+ upgradedDocumentCount: null,
3656
+ saved: savedBytesOfManifest(decoded.manifest)
3657
+ };
3320
3658
  }
3321
3659
 
3322
3660
  // src/persistence/durability/layout-transition/recovery.ts
@@ -3418,9 +3756,9 @@ async function settleStoredAnalysis(store, metadata, metadataOnly) {
3418
3756
  }
3419
3757
 
3420
3758
  // src/persistence/durability/recover-index.ts
3421
- async function writeCountedMetadata(deps, indexName, documentCount) {
3759
+ async function writeCountedMetadata(deps, indexName, documentCount, savedBytes) {
3422
3760
  await deps.queueMetadataWrite(indexName, async () => {
3423
- const metadata = deps.hooks.buildMetadata(indexName, documentCount);
3761
+ const metadata = deps.hooks.buildMetadata(indexName, documentCount, savedBytes);
3424
3762
  if (metadata === void 0) return;
3425
3763
  const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
3426
3764
  await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
@@ -3515,8 +3853,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3515
3853
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
3516
3854
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
3517
3855
  }
3856
+ await deps.measureStoredBytes(indexName, recovered.saved);
3518
3857
  if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
3519
- await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
3858
+ await writeCountedMetadata(
3859
+ deps,
3860
+ indexName,
3861
+ recovered.upgradedDocumentCount ?? manager.countDocuments(),
3862
+ recovered.saved.bytes
3863
+ );
3520
3864
  }
3521
3865
  return layout.transition === null ? null : removeUnread;
3522
3866
  }
@@ -3532,15 +3876,22 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
3532
3876
 
3533
3877
  // src/persistence/durability/manager.ts
3534
3878
  function createDurabilityManager(config, hooks, directoryOverride) {
3535
- const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
3879
+ const sizes = createStoredSizesRecord(true);
3880
+ const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
3881
+ const { directory } = logBytes;
3536
3882
  const canOffloadCheckpoint = directoryOverride === void 0;
3537
3883
  const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
3538
3884
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
3539
3885
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
3540
3886
  const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
3887
+ const rewriteBytesPerCheckpoint = config.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT;
3541
3888
  const mode = config.mode ?? "sync";
3542
3889
  const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
3543
- const registry = createPartitionRegistry(directory, segmentMaxBytes);
3890
+ const registry = createPartitionRegistry(
3891
+ directory,
3892
+ segmentMaxBytes,
3893
+ (indexName) => hooks.linkStoredSizes(indexName, sizes.hold(indexName))
3894
+ );
3544
3895
  const indexes = registry.indexes;
3545
3896
  const getOrCreateIndexState = registry.indexState;
3546
3897
  const getOrCreatePartition = registry.partition;
@@ -3631,12 +3982,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3631
3982
  indexName,
3632
3983
  indexState,
3633
3984
  compactionThreshold,
3985
+ rewriteBytesPerCheckpoint,
3634
3986
  canOffload: canOffloadCheckpoint,
3635
3987
  fromMemory,
3636
3988
  mutationThreshold: checkpointMutationThreshold,
3637
3989
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3638
3990
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
3639
3991
  keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
3992
+ recordSavedBytes: (saved) => sizes.setSaved(indexName, saved),
3640
3993
  queueMetadataWrite,
3641
3994
  markFatal,
3642
3995
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -3691,28 +4044,22 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3691
4044
  startAsyncFlushTimer();
3692
4045
  return outcomes;
3693
4046
  }
3694
- const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
4047
+ const { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps } = createIndexStorageAccess({
3695
4048
  directory,
4049
+ logBytes,
4050
+ sizes,
3696
4051
  hooks,
3697
4052
  indexes,
3698
4053
  metadataWrites,
3699
4054
  heldCheckpointFiles,
3700
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3701
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3702
- markFatal,
3703
- closeWriterReportingFailure
3704
- });
3705
- const recoveryDeps = {
3706
- directory,
3707
- hooks,
3708
4055
  compactionThreshold,
4056
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3709
4057
  registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
3710
4058
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
3711
4059
  },
3712
- holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
3713
- queueMetadataWrite,
3714
- syncStorage
3715
- };
4060
+ markFatal,
4061
+ closeWriterReportingFailure
4062
+ });
3716
4063
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3717
4064
  const recovered = recoverWithLayoutCheckpoint(
3718
4065
  recoveryDeps,
@@ -3809,6 +4156,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3809
4156
  syncStorage: () => syncStorage,
3810
4157
  ...queryHistoryAccess({ kind: "directory", directory }),
3811
4158
  ...archiveAccess,
4159
+ ...storedBytesAccess,
3812
4160
  async shutdown() {
3813
4161
  shuttingDown = true;
3814
4162
  stopTimers();
@@ -3827,6 +4175,31 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3827
4175
  };
3828
4176
  }
3829
4177
 
4178
+ // src/persistence/durability/snapshot-only/index-state.ts
4179
+ function createSnapshotIndexState() {
4180
+ return {
4181
+ mutationsSinceCheckpoint: 0,
4182
+ checkpointInFlight: null,
4183
+ applyChain: Promise.resolve(),
4184
+ unloading: false,
4185
+ appliedSeqNoByPartition: /* @__PURE__ */ new Map()
4186
+ };
4187
+ }
4188
+ function metadataKey(indexName) {
4189
+ return `${indexName}/meta`;
4190
+ }
4191
+ function noteSnapshotCommitted(state, positions, partitionCount) {
4192
+ let recordsLeft = 0;
4193
+ for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
4194
+ if (partitionId >= partitionCount) {
4195
+ state.appliedSeqNoByPartition.delete(partitionId);
4196
+ continue;
4197
+ }
4198
+ recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
4199
+ }
4200
+ state.mutationsSinceCheckpoint = recordsLeft;
4201
+ }
4202
+
3830
4203
  // src/persistence/durability/snapshot-only/checkpoint.ts
3831
4204
  function capture(input, prior, cutoffs) {
3832
4205
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
@@ -3885,10 +4258,21 @@ async function writeCaptured(input, store, prior, captured) {
3885
4258
  });
3886
4259
  const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
3887
4260
  recordWrittenVectors(input.vectorIndexes, vectors.written);
3888
- await removeCheckpointGarbage(
3889
- store,
3890
- await garbageThatNoFieldStillReads(store, input.vectorIndexes, committed.garbage)
3891
- );
4261
+ await input.onCommitted({
4262
+ saved: committed.saved,
4263
+ documentCount: captured.documentCount,
4264
+ partitionCount: captured.partitionCount
4265
+ });
4266
+ const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
4267
+ await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
4268
+ }
4269
+ async function saveCheckpointMetadata(adapter, hooks, indexName, committed) {
4270
+ const { saved, documentCount, partitionCount } = committed;
4271
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, saved.bytes);
4272
+ if (checkpointMetadata === void 0) return;
4273
+ checkpointMetadata.partitionCount = partitionCount;
4274
+ await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(checkpointMetadata, { checksum: true }));
4275
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, saved.bytes);
3892
4276
  }
3893
4277
  async function writeSnapshotOnlyCheckpoint(input) {
3894
4278
  const store = adapterCheckpointStore(input.adapter);
@@ -3899,41 +4283,12 @@ async function writeSnapshotOnlyCheckpoint(input) {
3899
4283
  });
3900
4284
  try {
3901
4285
  await writeCaptured(input, store, prior, captured);
3902
- return {
3903
- documentCount: captured.documentCount,
3904
- positions: captured.positions,
3905
- partitionCount: captured.partitionCount
3906
- };
4286
+ return { positions: captured.positions, partitionCount: captured.partitionCount };
3907
4287
  } finally {
3908
4288
  for (const plan of captured.plans.values()) plan.release();
3909
4289
  }
3910
4290
  }
3911
4291
 
3912
- // src/persistence/durability/snapshot-only/index-state.ts
3913
- function createSnapshotIndexState() {
3914
- return {
3915
- mutationsSinceCheckpoint: 0,
3916
- checkpointInFlight: null,
3917
- applyChain: Promise.resolve(),
3918
- unloading: false,
3919
- appliedSeqNoByPartition: /* @__PURE__ */ new Map()
3920
- };
3921
- }
3922
- function metadataKey(indexName) {
3923
- return `${indexName}/meta`;
3924
- }
3925
- function noteSnapshotCommitted(state, positions, partitionCount) {
3926
- let recordsLeft = 0;
3927
- for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
3928
- if (partitionId >= partitionCount) {
3929
- state.appliedSeqNoByPartition.delete(partitionId);
3930
- continue;
3931
- }
3932
- recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
3933
- }
3934
- state.mutationsSinceCheckpoint = recordsLeft;
3935
- }
3936
-
3937
4292
  // src/persistence/durability/snapshot-only/restores.ts
3938
4293
  function toError5(error) {
3939
4294
  return error instanceof Error ? error : new Error(String(error));
@@ -3971,15 +4326,67 @@ function createPendingRestores(onFatalError) {
3971
4326
  };
3972
4327
  }
3973
4328
 
4329
+ // src/persistence/durability/snapshot-only/saved-size.ts
4330
+ function createSnapshotSavedSize(adapter, settleRestore) {
4331
+ const sizes = createStoredSizesRecord(false);
4332
+ const generations = /* @__PURE__ */ new Map();
4333
+ const store = adapterCheckpointStore(adapter);
4334
+ function replace(indexName, saved) {
4335
+ const generation = (generations.get(indexName) ?? 0) + 1;
4336
+ generations.set(indexName, generation);
4337
+ sizes.setSaved(indexName, saved);
4338
+ sizes.setMeasuring(indexName, saved === null);
4339
+ return generation;
4340
+ }
4341
+ async function measureAfterRestore(indexName, generation) {
4342
+ await settleRestore(indexName);
4343
+ const decoded = await readSegmentManifest(store, indexName);
4344
+ if (generations.get(indexName) !== generation) return;
4345
+ replace(indexName, decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED);
4346
+ }
4347
+ return {
4348
+ record(indexName, saved) {
4349
+ replace(indexName, saved);
4350
+ },
4351
+ hold: sizes.hold,
4352
+ forget(indexName) {
4353
+ generations.delete(indexName);
4354
+ sizes.forget(indexName);
4355
+ },
4356
+ drop(indexName) {
4357
+ generations.delete(indexName);
4358
+ sizes.drop(indexName);
4359
+ },
4360
+ access: {
4361
+ storedSizesOf: sizes.of,
4362
+ addStoredSizes(indexName, stats) {
4363
+ sizes.of(indexName).addTo(stats);
4364
+ },
4365
+ vectorSavedBytes(indexName) {
4366
+ return sizes.of(indexName).vectorSavedBytes;
4367
+ },
4368
+ async measureRestoredStorage(indexName, restored) {
4369
+ if (restored !== void 0) {
4370
+ replace(indexName, restored);
4371
+ return;
4372
+ }
4373
+ const generation = replace(indexName, null);
4374
+ void measureAfterRestore(indexName, generation).catch(() => void 0);
4375
+ }
4376
+ }
4377
+ };
4378
+ }
4379
+
3974
4380
  // src/persistence/durability/snapshot-only/index.ts
3975
4381
  function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3976
4382
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
3977
4383
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
3978
4384
  const indexes = /* @__PURE__ */ new Map();
3979
- const metadataWrites = /* @__PURE__ */ new Map();
4385
+ const metadataWrites = createMetadataWriteQueue();
3980
4386
  const loads = createIndexLoads(hooks.getVectorIndexes);
3981
4387
  const restoreStorage = adapterRestoreStorage(adapter);
3982
4388
  const restores = createPendingRestores(hooks.onFatalError);
4389
+ const savedSize = createSnapshotSavedSize(adapter, settleRestore);
3983
4390
  let checkpointTimer = null;
3984
4391
  let shuttingDown = false;
3985
4392
  function settleRestore(indexName) {
@@ -3990,6 +4397,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3990
4397
  if (state === void 0) {
3991
4398
  state = createSnapshotIndexState();
3992
4399
  indexes.set(indexName, state);
4400
+ hooks.linkStoredSizes(indexName, savedSize.hold(indexName));
3993
4401
  }
3994
4402
  return state;
3995
4403
  }
@@ -4018,23 +4426,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4018
4426
  const manager = hooks.getManager(indexName);
4019
4427
  const metadata = hooks.buildTermsMetadata(indexName);
4020
4428
  if (manager === void 0 || metadata === void 0) return;
4021
- const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
4429
+ const { positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
4022
4430
  adapter,
4023
4431
  indexName,
4024
4432
  metadata,
4025
4433
  manager,
4026
4434
  vectorIndexes: hooks.getVectorIndexes(indexName),
4027
4435
  appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
4028
- rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
4029
- });
4030
- hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
4031
- await queueMetadataWrite(indexName, async () => {
4032
- const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
4033
- if (checkpointMetadata === void 0) return;
4034
- checkpointMetadata.partitionCount = partitionCount;
4035
- const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
4036
- await adapter.save(metadataKey(indexName), metadataBytes);
4037
- hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
4436
+ rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
4437
+ onCommitted: async (committed) => {
4438
+ savedSize.record(indexName, committed.saved);
4439
+ hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
4440
+ await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
4441
+ }
4038
4442
  });
4039
4443
  noteSnapshotCommitted(indexState, positions, partitionCount);
4040
4444
  if (publisher !== void 0) {
@@ -4060,16 +4464,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4060
4464
  );
4061
4465
  }
4062
4466
  function queueMetadataWrite(indexName, write) {
4063
- const previous = metadataWrites.get(indexName) ?? Promise.resolve();
4064
- const queued = previous.then(() => settleRestore(indexName)).then(write);
4065
- metadataWrites.set(
4066
- indexName,
4067
- queued.catch(() => void 0)
4068
- );
4069
- return queued;
4467
+ return metadataWrites.queue(indexName, () => settleRestore(indexName).then(write));
4070
4468
  }
4071
- async function saveMetadata(indexName, documentCount) {
4072
- const metadata = hooks.buildMetadata(indexName, documentCount);
4469
+ async function saveMetadata(indexName, documentCount, savedBytes) {
4470
+ const metadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
4073
4471
  if (metadata === void 0) return;
4074
4472
  await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
4075
4473
  }
@@ -4079,9 +4477,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4079
4477
  async function loadIndex(indexName, metadataOnly) {
4080
4478
  await settleRestore(indexName);
4081
4479
  const metaBytes = await adapter.load(metadataKey(indexName));
4082
- if (metaBytes === null) {
4083
- return;
4084
- }
4480
+ if (metaBytes === null) return;
4085
4481
  const { metadata } = await readMetadataEnvelope(metaBytes);
4086
4482
  const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
4087
4483
  let derivedCount;
@@ -4120,6 +4516,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4120
4516
  onLayoutUpgrade: hooks.onLayoutUpgrade
4121
4517
  });
4122
4518
  const indexState = getOrCreateIndexState(indexName);
4519
+ savedSize.record(indexName, recovered.saved);
4123
4520
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
4124
4521
  const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
4125
4522
  if (persistedSeqNo > 0) {
@@ -4128,7 +4525,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4128
4525
  }
4129
4526
  if (recovered.upgradedDocumentCount !== null || promoted) {
4130
4527
  const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
4131
- await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
4528
+ await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount, recovered.saved.bytes)).catch(
4529
+ () => void 0
4530
+ );
4132
4531
  }
4133
4532
  }
4134
4533
  async function recordMutation(record) {
@@ -4160,14 +4559,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4160
4559
  try {
4161
4560
  await indexState.applyChain;
4162
4561
  if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
4163
- await metadataWrites.get(indexName);
4562
+ await metadataWrites.pending(indexName);
4164
4563
  } catch (error) {
4165
4564
  indexState.unloading = false;
4166
4565
  throw error;
4167
4566
  }
4168
4567
  }
4169
4568
  indexes.delete(indexName);
4170
- metadataWrites.delete(indexName);
4569
+ metadataWrites.forget(indexName);
4570
+ savedSize.forget(indexName);
4171
4571
  }
4172
4572
  return {
4173
4573
  isActive() {
@@ -4249,6 +4649,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4249
4649
  }
4250
4650
  },
4251
4651
  ...queryHistoryAccess({ kind: "adapter", adapter }),
4652
+ ...savedSize.access,
4252
4653
  async removeIndex(indexName) {
4253
4654
  await settleRestore(indexName).catch(() => void 0);
4254
4655
  restores.forget(indexName);
@@ -4258,6 +4659,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4258
4659
  await adapter.delete(key);
4259
4660
  }
4260
4661
  await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
4662
+ savedSize.drop(indexName);
4261
4663
  },
4262
4664
  async unloadIndex(indexName) {
4263
4665
  await settleRestore(indexName);
@@ -4287,7 +4689,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4287
4689
  }
4288
4690
 
4289
4691
  // src/engine/durability-integration.ts
4290
- function buildMetadata(indexName, hooks, documentCount) {
4692
+ function buildMetadata(indexName, hooks, documentCount, savedBytes) {
4291
4693
  const config = hooks.getIndexConfig(indexName);
4292
4694
  if (config === void 0) {
4293
4695
  return void 0;
@@ -4297,7 +4699,8 @@ function buildMetadata(indexName, hooks, documentCount) {
4297
4699
  indexName,
4298
4700
  config,
4299
4701
  manager === void 0 ? void 0 : metadataPartitionCount(manager),
4300
- documentCount
4702
+ documentCount,
4703
+ savedBytes
4301
4704
  );
4302
4705
  }
4303
4706
  function createDurabilityIntegration(tier, hooks) {
@@ -4305,7 +4708,7 @@ function createDurabilityIntegration(tier, hooks) {
4305
4708
  getManager: hooks.getManager,
4306
4709
  getVectorFieldPaths: hooks.getVectorFieldPaths,
4307
4710
  getVectorIndexes: hooks.getVectorIndexes,
4308
- buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
4711
+ buildMetadata: (indexName, documentCount, savedBytes) => buildMetadata(indexName, hooks, documentCount, savedBytes),
4309
4712
  buildTermsMetadata(indexName) {
4310
4713
  const config = hooks.getTermsConfig(indexName);
4311
4714
  return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
@@ -4320,7 +4723,8 @@ function createDurabilityIntegration(tier, hooks) {
4320
4723
  onCheckpointHeld: hooks.onCheckpointHeld,
4321
4724
  onFatalError: hooks.onFatalError,
4322
4725
  ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
4323
- ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4726
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved },
4727
+ linkStoredSizes: hooks.linkStoredSizes
4324
4728
  };
4325
4729
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4326
4730
  async function recordMutation(indexName, docId, operation, document, apply) {
@@ -4574,7 +4978,7 @@ function createInvalidationIntegration(deps) {
4574
4978
 
4575
4979
  // src/engine/durability-wiring.ts
4576
4980
  function createDurabilityFromTier(tier, wiring) {
4577
- return createDurabilityIntegration(tier, {
4981
+ const integration = createDurabilityIntegration(tier, {
4578
4982
  checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
4579
4983
  getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
4580
4984
  getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
@@ -4594,8 +4998,25 @@ function createDurabilityFromTier(tier, wiring) {
4594
4998
  },
4595
4999
  queueDiskAdoption: wiring.queueDiskAdoption,
4596
5000
  awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
4597
- createIndexFromMetadata: wiring.createIndexFromMetadata,
4598
- recordCheckpoint: wiring.recordCheckpoint,
5001
+ async createIndexFromMetadata(metadata, loadData) {
5002
+ await wiring.createIndexFromMetadata(metadata, loadData);
5003
+ const entry = wiring.indexRegistry.get(metadata.indexName);
5004
+ if (entry === void 0) return;
5005
+ entry.storedSizes = integration.manager.storedSizesOf(metadata.indexName);
5006
+ entry.storedSizes.recordSavedBytes(entry.savedBytes);
5007
+ },
5008
+ recordCheckpoint(indexName, documentCount, partitionCount, savedBytes) {
5009
+ const entry = wiring.indexRegistry.get(indexName);
5010
+ if (entry === void 0) return;
5011
+ entry.documentCount = documentCount;
5012
+ entry.partitionCount = partitionCount;
5013
+ entry.savedBytes = savedBytes;
5014
+ entry.storedSizes?.recordSavedBytes(savedBytes);
5015
+ },
5016
+ linkStoredSizes(indexName, sizes) {
5017
+ const entry = wiring.indexRegistry.get(indexName);
5018
+ if (entry !== void 0) entry.storedSizes = sizes;
5019
+ },
4599
5020
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
4600
5021
  onCheckpointHeld: wiring.emitCheckpointHeld,
4601
5022
  onFatalError: wiring.emitFatalError,
@@ -4609,6 +5030,7 @@ function createDurabilityFromTier(tier, wiring) {
4609
5030
  }
4610
5031
  }
4611
5032
  });
5033
+ return integration;
4612
5034
  }
4613
5035
  async function reloadIndexAndCopies(orchestrator, durability, indexName) {
4614
5036
  const manager = durability?.manager;
@@ -5456,8 +5878,9 @@ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
5456
5878
  partitionCount ?? void 0
5457
5879
  );
5458
5880
  }
5459
- function restoredRegistryEntry(prepared, manager, deps) {
5881
+ function restoredRegistryEntry(indexName, prepared, manager, deps) {
5460
5882
  return {
5883
+ storedSizes: deps.durability?.manager.storedSizesOf(indexName),
5461
5884
  config: prepared.config,
5462
5885
  language: prepared.language,
5463
5886
  embeddingAdapter: prepared.embeddingAdapter,
@@ -5557,13 +5980,17 @@ async function commitToStorage(indexName, restored, previous, deps) {
5557
5980
  for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
5558
5981
  await staging.commit();
5559
5982
  } catch (error) {
5560
- if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
5983
+ if (previousPartitions > 0) {
5984
+ durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
5985
+ await durable.measureRestoredStorage?.(indexName).catch(() => void 0);
5986
+ }
5561
5987
  throw error;
5562
5988
  }
5563
5989
  for (const paged of pagedSegmentsOf(previous)) paged.close();
5564
5990
  const restoredSegments = pagedSegmentsOf(restored.detached.manager);
5565
5991
  for (const paged of restoredSegments) paged.suspend();
5566
5992
  await staging.finish();
5993
+ await durable.measureRestoredStorage?.(indexName, restored.saved ?? void 0).catch(() => void 0);
5567
5994
  const relocate = await staging.relocation().catch(() => null);
5568
5995
  for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
5569
5996
  if (relocate === null) return;
@@ -5572,6 +5999,7 @@ async function commitToStorage(indexName, restored, previous, deps) {
5572
5999
  function activate(indexName, restored, entry, deps) {
5573
6000
  deps.executor.installDetachedIndex(restored.detached);
5574
6001
  deps.indexRegistry.set(indexName, entry);
6002
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
5575
6003
  restored.detached.attach();
5576
6004
  deps.clearAnalysisStale(indexName);
5577
6005
  const { prepared } = restored;
@@ -5611,6 +6039,7 @@ async function createFresh(indexName, restored, entry, deps) {
5611
6039
  deps.copies.desyncIndex(indexName);
5612
6040
  deps.executor.installDetachedIndex(restored.detached);
5613
6041
  deps.indexRegistry.set(indexName, entry);
6042
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
5614
6043
  try {
5615
6044
  await deps.indexState.registerReplacing(indexName, async () => {
5616
6045
  await commitToStorage(indexName, restored, void 0, deps);
@@ -5629,9 +6058,10 @@ async function stageMetadata(indexName, staging, entry) {
5629
6058
  await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
5630
6059
  }
5631
6060
  async function installRestoredIndex(indexName, restored, deps, history = null) {
5632
- const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
6061
+ const entry = restoredRegistryEntry(indexName, restored.prepared, restored.detached.manager, deps);
5633
6062
  const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
5634
6063
  if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
6064
+ if (restored.saved !== null) entry.savedBytes = restored.saved.bytes;
5635
6065
  let replaced;
5636
6066
  try {
5637
6067
  if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
@@ -5764,7 +6194,7 @@ async function loadStagedIndex(indexName, staged, staging, store, deps) {
5764
6194
  deps.executor.discardDetachedIndex(detached);
5765
6195
  throw error;
5766
6196
  }
5767
- return { prepared, detached, staging, checkpoint: manifest.checkpoint };
6197
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint, saved: savedBytesOfManifest(manifest) };
5768
6198
  }
5769
6199
  async function restoreArchive(indexName, reader, deps) {
5770
6200
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
@@ -5865,7 +6295,7 @@ function insertSnapshotDocuments(envelope, manager) {
5865
6295
  for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
5866
6296
  }
5867
6297
  async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
5868
- const entry = restoredRegistryEntry(prepared, built.manager, deps);
6298
+ const entry = restoredRegistryEntry(indexName, prepared, built.manager, deps);
5869
6299
  const archive = await buildArchiveFromMemory({
5870
6300
  indexName,
5871
6301
  metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
@@ -5889,7 +6319,7 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
5889
6319
  let manifest;
5890
6320
  try {
5891
6321
  insertSnapshotDocuments(envelope, built.manager);
5892
- if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
6322
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [], saved: null };
5893
6323
  manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
5894
6324
  } catch (err) {
5895
6325
  deps.executor.discardDetachedIndex(built);