@delali/narsil 0.3.1-next.180 → 0.3.1-next.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
  4. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
  5. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  6. package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
  7. package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
  8. package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
  9. package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
  10. package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
  11. package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
  12. package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
  13. package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
  14. package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
  15. package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
  16. package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
  17. package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
  18. package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
  19. package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
  20. package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
  21. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  22. package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
  23. package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
  24. package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
  25. package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
  26. package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
  27. package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
  28. package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
  29. package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
  30. package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
  31. package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
  32. package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
  33. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  34. package/dist/client.d.ts +6 -6
  35. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  36. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  37. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
  38. package/dist/distribution.d.ts +4 -4
  39. package/dist/distribution.mjs +391 -138
  40. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  41. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  42. package/dist/index.browser.mjs +1 -1
  43. package/dist/index.d.ts +4 -4
  44. package/dist/index.mjs +1 -1
  45. package/dist/index.react-native.mjs +1 -1
  46. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
  47. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
  48. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
  49. package/dist/partial-words.browser.mjs +1 -1
  50. package/dist/partial-words.mjs +1 -1
  51. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  52. package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
  53. package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
  54. package/dist/query-suggestions/merge-worker.mjs +4 -4
  55. package/dist/query-suggestions.browser.mjs +1 -1
  56. package/dist/query-suggestions.d.ts +1 -1
  57. package/dist/query-suggestions.mjs +2 -2
  58. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
  59. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
  60. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
  61. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
  62. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
  63. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
  64. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  65. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
  66. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
  67. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
  68. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
  69. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
  70. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
  71. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
  72. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
  73. package/dist/react-native/partial-words.mjs +1 -1
  74. package/dist/react-native/query-suggestions.mjs +1 -1
  75. package/dist/react-native/sync.mjs +1 -1
  76. package/dist/react-native/synonyms.mjs +1 -1
  77. package/dist/react.d.ts +7 -7
  78. package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
  79. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
  80. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
  81. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
  82. package/dist/server.d.ts +8 -8
  83. package/dist/server.mjs +16 -16
  84. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
  85. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
  86. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
  87. package/dist/sync/filesystem.d.ts +2 -2
  88. package/dist/sync/filesystem.mjs +6 -6
  89. package/dist/sync/indexeddb.d.ts +2 -2
  90. package/dist/sync.browser.mjs +1 -1
  91. package/dist/sync.d.ts +3 -3
  92. package/dist/sync.mjs +1 -1
  93. package/dist/synonyms.browser.mjs +1 -1
  94. package/dist/synonyms.mjs +1 -1
  95. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  96. package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
  97. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  98. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
  99. package/dist/workers/entry.mjs +7 -7
  100. package/openapi.json +32 -0
  101. package/package.json +7 -7
@@ -1,10 +1,10 @@
1
1
  import { sha256 } from './chunk-H3MDVPBO.mjs';
2
2
  import { oneSyncChangeAtATime } from './chunk-Q32DDFKX.mjs';
3
- import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, packChecksummedEnvelope, hasNrslMagic, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_FOLDER, packEnvelopeWithChecksum, checksumOfChunks, storedStateToWire, storedStateFromWire } from './chunk-F6BJYCF2.mjs';
4
- import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-57R74JWC.mjs';
5
- import { deserializePartition } from './chunk-H344ZXRH.mjs';
6
- import { mergeFieldIndexes, selectionKeeps, fieldIndexRemapOf, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, shareSegmentArrays, freezeSegmentShared, holdsADiscardedField, toError, segmentArraysOf, mergeSegmentContents, buildSegmentPayloads, createRequestId } from './chunk-GGBATERA.mjs';
7
- import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, loadSchemaCodeFor, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, isCompositePartition, yieldToEventLoop, decodeStoredDocument, buildPartialWordTableFrom, configurationPayloadOf, withOpenIndex, emitSyncChange, newSyncEpoch, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, prepareDocumentVectors, removedSurfaceSpellings, prepareConfiguredSynonyms, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, mergeKeywordSources, createSharedFrozenSegment, ARRAY_POSITION_GAP, createFrozenSegment, createPublicationGate, prepareRemovalText, prepareDocumentText, writePartitionDocument, updateDocumentVectors, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, readHeapStatistics, readResidentMemory, holdSyncEpochChange, emptyPendingRegions, mergePostings2, scanPostingBlocks, SegmentRefusal, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, endOfValue, compareScores, isSyncEpoch, decodeEveryStoredDocument, loadGzip, createPartitionManager, createPartitionRouter, arrayPositionsAreStale, insertDocumentVectors, decodeMapWithoutFields, survivorDocumentsOf, partitionStateOf, removeDocumentVectors } from './chunk-F4MIGBBP.mjs';
3
+ import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, packChecksummedEnvelope, hasNrslMagic, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_FOLDER, packEnvelopeWithChecksum, checksumOfChunks, storedStateToWire, storedStateFromWire } from './chunk-UPN4NUXU.mjs';
4
+ import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-G4YYO5TC.mjs';
5
+ import { deserializePartition } from './chunk-DN2GVVR4.mjs';
6
+ import { mergeFieldIndexes, selectionKeeps, fieldIndexRemapOf, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, shareSegmentArrays, freezeSegmentShared, holdsADiscardedField, toError, segmentArraysOf, mergeSegmentContents, buildSegmentPayloads, createRequestId } from './chunk-XNIH3WIB.mjs';
7
+ import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, loadSchemaCodeFor, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, isCompositePartition, yieldToEventLoop, decodeStoredDocument, buildPartialWordTableFrom, configurationPayloadOf, withOpenIndex, emitSyncChange, newSyncEpoch, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, prepareDocumentVectors, removedSurfaceSpellings, prepareConfiguredSynonyms, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, mergeKeywordSources, createSharedFrozenSegment, ARRAY_POSITION_GAP, createFrozenSegment, createPublicationGate, prepareRemovalText, prepareDocumentText, writePartitionDocument, updateDocumentVectors, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, readHeapStatistics, readResidentMemory, holdSyncEpochChange, emptyPendingRegions, mergePostings2, scanPostingBlocks, SegmentRefusal, DEFAULT_BM25_B, DEFAULT_BM25_K1, VERSION, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, endOfValue, compareScores, isSyncEpoch, decodeEveryStoredDocument, loadGzip, createPartitionManager, createPartitionRouter, arrayPositionsAreStale, insertDocumentVectors, decodeMapWithoutFields, survivorDocumentsOf, partitionStateOf, removeDocumentVectors } from './chunk-6A5XE2MA.mjs';
8
8
  import './chunk-J2BQTJJE.mjs';
9
9
  import { DEFAULT_MAX_PARTITIONS, isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, MAX_PARTITIONS, STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, countWrite, ANALYSIS_CHANGE_MEMBERS, collectAndReturnPages, requireBoundedUpperLists, osqRecordBytes, createFixedBuffer, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, createVectorFileReader, MAX_IMPORT_REFUSALS_REPORTED, decodeVectorIndexParts, signalOfSearch, requireSearchActive, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, decodeVectorFilePayload, DEFAULT_VECTOR_METRIC, defaultQuantizationFor, VECTOR_FILE_MAX_VECTORS, deadBytesFor, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, vectorsPerFile, vectorsToBytes, decodeVectorGraphPartPayload, assembleGraphParts, nodeThreadId, fnv1a, osqBitsOf, countDead, detectRuntime, graphPartsOfEarlierPayload, decodeVectorGraphPayload } from './chunk-I7DTUOK2.mjs';
10
10
  export { createHNSWIndex, osqBitsOf } from './chunk-I7DTUOK2.mjs';
@@ -1669,6 +1669,8 @@ var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
1669
1669
  var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
1670
1670
  var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
1671
1671
  var LOG_RECORDS_PER_CHECKPOINT = 1e5;
1672
+ var REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE = 0.2;
1673
+ var DEFAULT_REWRITE_BYTES_PER_CHECKPOINT = 268435456;
1672
1674
  var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
1673
1675
  var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
1674
1676
  var CHECKPOINT_WORKER_IDLE_MS = 5e3;
@@ -2533,6 +2535,9 @@ function metadataToWire(meta) {
2533
2535
  if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
2534
2536
  wire.document_count = meta.documentCount;
2535
2537
  }
2538
+ if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
2539
+ wire.saved_bytes = meta.savedBytes;
2540
+ }
2536
2541
  if (meta.embedding) {
2537
2542
  wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
2538
2543
  }
@@ -2648,6 +2653,9 @@ function wireToMetadata(raw) {
2648
2653
  if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
2649
2654
  meta.documentCount = raw.document_count;
2650
2655
  }
2656
+ if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
2657
+ meta.savedBytes = raw.saved_bytes;
2658
+ }
2651
2659
  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
2652
2660
  meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
2653
2661
  }
@@ -4018,6 +4026,12 @@ function createReadCache(capacity) {
4018
4026
  function evictDownTo(limit) {
4019
4027
  for (const [key, entry] of entries) {
4020
4028
  if (held <= limit) return;
4029
+ if (entry.referenced) {
4030
+ entry.referenced = false;
4031
+ entries.delete(key);
4032
+ entries.set(key, entry);
4033
+ continue;
4034
+ }
4021
4035
  drop(key, entry);
4022
4036
  }
4023
4037
  }
@@ -4072,8 +4086,7 @@ function createReadCache(capacity) {
4072
4086
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
4073
4087
  const found = entries.get(key);
4074
4088
  if (found === void 0) return void 0;
4075
- entries.delete(key);
4076
- entries.set(key, found);
4089
+ found.referenced = true;
4077
4090
  return found.value;
4078
4091
  },
4079
4092
  put(fileNumber, kind, entry, value, bytes) {
@@ -4081,7 +4094,7 @@ function createReadCache(capacity) {
4081
4094
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
4082
4095
  const existing = entries.get(key);
4083
4096
  if (existing !== void 0) drop(key, existing);
4084
- entries.set(key, { value, bytes, fileNumber });
4097
+ entries.set(key, { value, bytes, fileNumber, referenced: true });
4085
4098
  held += bytes;
4086
4099
  perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
4087
4100
  evictDownTo(capacity - reserved);
@@ -5381,28 +5394,55 @@ async function pagedListingOf(directory, ref) {
5381
5394
  function yieldToEventLoop2() {
5382
5395
  return new Promise((resolve) => setTimeout(resolve, 0));
5383
5396
  }
5397
+ function newerTombstonesOf(prepared, at) {
5398
+ const docIds = /* @__PURE__ */ new Set();
5399
+ for (let newer = at + 1; newer < prepared.length; newer++) {
5400
+ const access = prepared[newer].segment.paged;
5401
+ if (access === null) continue;
5402
+ for (const docId of access.snapshot.tombstones) docIds.add(docId);
5403
+ }
5404
+ return docIds;
5405
+ }
5406
+ function scansEveryId(segment, tombstoned) {
5407
+ const held = segment.arrays.idTable.count;
5408
+ return tombstoned.size * Math.ceil(Math.log2(held + 1)) > held;
5409
+ }
5410
+ function tombstoneScannedOrdinal(segment, tombstoned, ordinal) {
5411
+ const docId = segment.arrays.idTable.idAt(ordinal);
5412
+ if (tombstoned.has(docId)) segment.prepareTombstone(docId, void 0, void 0, ordinal)?.publish();
5413
+ }
5384
5414
  function applyNewerTombstones(prepared) {
5385
5415
  for (let at = 0; at < prepared.length; at++) {
5386
5416
  if (!prepared[at].isNew) continue;
5387
- for (let newer = at + 1; newer < prepared.length; newer++) {
5388
- const access = prepared[newer].segment.paged;
5389
- if (access === null) continue;
5390
- for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
5417
+ const { segment } = prepared[at];
5418
+ const tombstoned = newerTombstonesOf(prepared, at);
5419
+ if (!scansEveryId(segment, tombstoned)) {
5420
+ for (const docId of tombstoned) segment.tombstoneDocument(docId);
5421
+ continue;
5391
5422
  }
5423
+ const held = segment.arrays.idTable.count;
5424
+ for (let ordinal = 0; ordinal < held; ordinal++) tombstoneScannedOrdinal(segment, tombstoned, ordinal);
5392
5425
  }
5393
5426
  }
5394
5427
  async function applyNewerTombstonesInTurns(prepared) {
5395
- let applied = 0;
5428
+ let steps = 0;
5396
5429
  for (let at = 0; at < prepared.length; at++) {
5397
5430
  if (!prepared[at].isNew) continue;
5398
- for (let newer = at + 1; newer < prepared.length; newer++) {
5399
- const access = prepared[newer].segment.paged;
5400
- if (access === null) continue;
5401
- for (const docId of access.snapshot.tombstones) {
5402
- prepared[at].segment.tombstoneDocument(docId);
5403
- applied += 1;
5404
- if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5431
+ const { segment } = prepared[at];
5432
+ const tombstoned = newerTombstonesOf(prepared, at);
5433
+ if (!scansEveryId(segment, tombstoned)) {
5434
+ for (const docId of tombstoned) {
5435
+ segment.tombstoneDocument(docId);
5436
+ steps += 1;
5437
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5405
5438
  }
5439
+ continue;
5440
+ }
5441
+ const held = segment.arrays.idTable.count;
5442
+ for (let ordinal = 0; ordinal < held; ordinal++) {
5443
+ tombstoneScannedOrdinal(segment, tombstoned, ordinal);
5444
+ steps += 1;
5445
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5406
5446
  }
5407
5447
  }
5408
5448
  }
@@ -8076,6 +8116,9 @@ function wholePartitionsWhereMostChanged(manager, rules) {
8076
8116
  return whole;
8077
8117
  };
8078
8118
  }
8119
+ function liveDocumentsOf(manager, targets) {
8120
+ return targets.map(({ partitionId }) => ({ partitionId, documentCount: manager.getPartition(partitionId).count() }));
8121
+ }
8079
8122
  async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
8080
8123
  const whileNoBuildPassOrMutationRuns = (capture3) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture3));
8081
8124
  return whileNoBuildPassOrMutationRuns(
@@ -8118,7 +8161,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
8118
8161
  vectorPlans,
8119
8162
  wholePartitions,
8120
8163
  recordsLeftInLog,
8121
- unreplayedCutoffs: cutoffs
8164
+ unreplayedCutoffs: cutoffs,
8165
+ liveDocuments: memoryMatchesTheTargets && cutoffs.size === 0 ? liveDocumentsOf(manager, targets) : null
8122
8166
  };
8123
8167
  } catch (error) {
8124
8168
  for (const plan of vectorPlans.values()) plan.release();
@@ -8355,6 +8399,25 @@ async function pagedTombstonesFor(directory, older, written, deleted) {
8355
8399
  return [...tombstones];
8356
8400
  }
8357
8401
 
8402
+ // src/persistence/durability/stored-bytes.ts
8403
+ var NOTHING_SAVED = { bytes: 0, vectorBytes: 0, partitions: [] };
8404
+ function savedBytesOfManifest(manifest) {
8405
+ let bytes = 0;
8406
+ const partitions = [];
8407
+ for (const partition of manifest.partitions) {
8408
+ let partitionBytes = 0;
8409
+ for (const segment of partition.segments) partitionBytes += segment.bytes;
8410
+ partitions.push({ partitionId: partition.partitionId, bytes: partitionBytes });
8411
+ bytes += partitionBytes;
8412
+ }
8413
+ let vectorBytes = 0;
8414
+ for (const field of manifest.vectors) {
8415
+ for (const file of field.files) vectorBytes += file.bytes;
8416
+ for (const part of field.graphParts) vectorBytes += part.bytes;
8417
+ }
8418
+ return { bytes: bytes + vectorBytes, vectorBytes, partitions };
8419
+ }
8420
+
8358
8421
  // src/persistence/durability/segment/build-segment.ts
8359
8422
  function writeRecordsOf(documents, written) {
8360
8423
  const seqNos = new Float64Array(documents.length);
@@ -10300,6 +10363,9 @@ async function mergePagedWindow(input) {
10300
10363
  for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
10301
10364
  }
10302
10365
  }
10366
+ for (const docId of input.replacedAfterWindow ?? []) {
10367
+ for (const source of sources) source.segment.tombstoneDocument(docId);
10368
+ }
10303
10369
  const tombstones = /* @__PURE__ */ new Set();
10304
10370
  if (!input.dropsEveryTombstone) {
10305
10371
  for (const source of sources) {
@@ -10393,49 +10459,34 @@ function retainedTombstones(frozen, contents) {
10393
10459
  }
10394
10460
  return [...retained];
10395
10461
  }
10396
- async function compactPartitionSegments(input) {
10397
- const { segments, compactionThreshold } = input;
10398
- if (segments.length <= compactionThreshold) {
10399
- return { segments, nextSegmentId: input.nextSegmentId };
10400
- }
10401
- const paged = input.paged === true;
10402
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
10403
- if (window === null) {
10404
- return { segments, nextSegmentId: input.nextSegmentId };
10405
- }
10406
- const windowRefs = segments.slice(window.start, window.start + window.windowLength);
10407
- const replacing = (written2) => ({
10408
- segments: [
10409
- ...segments.slice(0, window.start),
10410
- ...written2.segments,
10411
- ...segments.slice(window.start + window.windowLength)
10412
- ],
10413
- nextSegmentId: written2.nextSegmentId
10414
- });
10415
- if (paged) {
10462
+ async function mergeSegmentWindow(input, window) {
10463
+ if (input.paged === true) {
10416
10464
  const { directory } = input.store;
10417
- if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
10418
- return replacing(
10419
- await mergePagedWindow({
10420
- store: input.store,
10421
- directory,
10422
- indexName: input.indexName,
10423
- partitionId: input.partitionId,
10424
- schema: input.schema,
10425
- partialWordAnalysis: input.partialWordAnalysis,
10426
- textAnalysis: input.textAnalysis,
10427
- nextSegmentId: input.nextSegmentId,
10428
- refs: windowRefs,
10429
- dropsEveryTombstone: window.start === 0
10430
- })
10431
- );
10465
+ if (directory === void 0) return null;
10466
+ return mergePagedWindow({
10467
+ store: input.store,
10468
+ directory,
10469
+ indexName: input.indexName,
10470
+ partitionId: input.partitionId,
10471
+ schema: input.schema,
10472
+ partialWordAnalysis: input.partialWordAnalysis,
10473
+ textAnalysis: input.textAnalysis,
10474
+ nextSegmentId: input.nextSegmentId,
10475
+ refs: window.refs,
10476
+ dropsEveryTombstone: window.startsTheList,
10477
+ ...window.replacedAfterWindow === void 0 ? {} : { replacedAfterWindow: window.replacedAfterWindow }
10478
+ });
10432
10479
  }
10433
10480
  const contents = [];
10434
- for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
10481
+ for (const ref of window.refs) contents.push(await readSegmentFile(input.store, ref));
10435
10482
  const frozen = contents.map(frozenOf);
10436
10483
  applyTimeOrder(frozen, contents);
10484
+ for (const docId of window.replacedAfterWindow ?? []) {
10485
+ for (const segment of frozen) segment.tombstoneDocument(docId);
10486
+ }
10437
10487
  const writeRecords = writeRecordsOf2(frozen, contents);
10438
- const written = await writeSegmentsOf({
10488
+ const keepsTombstones = !(window.startsTheList && window.replacedAfterWindow !== void 0);
10489
+ return writeSegmentsOf({
10439
10490
  store: input.store,
10440
10491
  indexName: input.indexName,
10441
10492
  partitionId: input.partitionId,
@@ -10444,11 +10495,33 @@ async function compactPartitionSegments(input) {
10444
10495
  partialWordAnalysis: input.partialWordAnalysis,
10445
10496
  textAnalysis: input.textAnalysis,
10446
10497
  selections: frozen.map((segment) => ({ segment })),
10447
- tombstones: retainedTombstones(frozen, contents),
10498
+ tombstones: keepsTombstones ? retainedTombstones(frozen, contents) : [],
10448
10499
  mergesIntoOneFile: true,
10449
10500
  ...writeRecords === void 0 ? {} : { writeRecords }
10450
10501
  });
10451
- return replacing(written);
10502
+ }
10503
+ async function compactPartitionSegments(input) {
10504
+ const { segments, compactionThreshold } = input;
10505
+ if (segments.length <= compactionThreshold) {
10506
+ return { segments, nextSegmentId: input.nextSegmentId };
10507
+ }
10508
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, input.paged === true);
10509
+ if (window === null) {
10510
+ return { segments, nextSegmentId: input.nextSegmentId };
10511
+ }
10512
+ const written = await mergeSegmentWindow(input, {
10513
+ refs: segments.slice(window.start, window.start + window.windowLength),
10514
+ startsTheList: window.start === 0
10515
+ });
10516
+ if (written === null) return { segments, nextSegmentId: input.nextSegmentId };
10517
+ return {
10518
+ segments: [
10519
+ ...segments.slice(0, window.start),
10520
+ ...written.segments,
10521
+ ...segments.slice(window.start + window.windowLength)
10522
+ ],
10523
+ nextSegmentId: written.nextSegmentId
10524
+ };
10452
10525
  }
10453
10526
 
10454
10527
  // src/persistence/durability/segment/manifest-encode.ts
@@ -10497,6 +10570,161 @@ function encodeSegmentManifest(manifest) {
10497
10570
  return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
10498
10571
  }
10499
10572
 
10573
+ // src/persistence/paged/segment-scan.ts
10574
+ async function openPagedSegmentScan(directory, ref) {
10575
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
10576
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10577
+ const segment = createPagedFrozenSegment(snapshot, cache);
10578
+ const { paged } = segment;
10579
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10580
+ const ids = segment.arrays.idTable;
10581
+ const tombstones = new Set(snapshot.tombstones);
10582
+ return {
10583
+ count: snapshot.documentCount,
10584
+ schema: snapshot.schema,
10585
+ tombstones,
10586
+ idAt: (ordinal) => ids.idAt(ordinal),
10587
+ holds: (docId) => ids.ordinalOf(docId) >= 0,
10588
+ storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
10589
+ close: () => {
10590
+ paged.close();
10591
+ cache.clear();
10592
+ }
10593
+ };
10594
+ }
10595
+ async function checkPagedSegmentFully(directory, ref) {
10596
+ const listing = await pagedListingOf(directory, ref);
10597
+ await requirePagedPayloadChecksum(directory, listing);
10598
+ const snapshot = openPagedSnapshot(listing);
10599
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10600
+ const segment = createPagedFrozenSegment(snapshot, cache);
10601
+ const { paged } = segment;
10602
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10603
+ try {
10604
+ await checkPagedContents(segment, cache);
10605
+ return snapshot.schema;
10606
+ } finally {
10607
+ paged.close();
10608
+ cache.clear();
10609
+ }
10610
+ }
10611
+ async function bytesThatSegmentsHoldInMemory(directory, manifest) {
10612
+ let bytes = 0;
10613
+ for (const partition of manifest.partitions) {
10614
+ for (const ref of partition.segments) {
10615
+ const paged = await isPagedSegmentFile(directory, ref.key);
10616
+ bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
10617
+ }
10618
+ }
10619
+ return bytes;
10620
+ }
10621
+ function countsAsLatestHolder(scans, at, docId) {
10622
+ for (let later = at + 1; later < scans.length; later++) {
10623
+ if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
10624
+ }
10625
+ return true;
10626
+ }
10627
+
10628
+ // src/persistence/durability/segment/removed-documents.ts
10629
+ var decoder2 = new TextDecoder();
10630
+ async function readSegmentIds(store, ref) {
10631
+ const { directory } = store;
10632
+ if (directory !== void 0 && await isPagedSegmentFile(directory, ref.key)) {
10633
+ const scan = await openPagedSegmentScan(directory, ref);
10634
+ try {
10635
+ const ids2 = new Array(scan.count);
10636
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) ids2[ordinal] = scan.idAt(ordinal);
10637
+ return { ids: ids2, tombstones: [...scan.tombstones] };
10638
+ } finally {
10639
+ scan.close();
10640
+ }
10641
+ }
10642
+ const contents = await readSegmentFile(store, ref);
10643
+ const table = contents.arrays.idTable;
10644
+ const ids = new Array(contents.arrays.documentCount);
10645
+ for (let ordinal = 0; ordinal < ids.length; ordinal++) {
10646
+ ids[ordinal] = decoder2.decode(table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]));
10647
+ }
10648
+ return { ids, tombstones: contents.tombstones };
10649
+ }
10650
+ async function documentsReplacedLater(store, segments) {
10651
+ const later = /* @__PURE__ */ new Set();
10652
+ const replaced = new Array(segments.length);
10653
+ for (let at = segments.length - 1; at >= 0; at--) {
10654
+ const { ids, tombstones } = await readSegmentIds(store, segments[at]);
10655
+ const found = [];
10656
+ for (const docId of ids) if (later.has(docId)) found.push(docId);
10657
+ replaced[at] = found;
10658
+ if (at === 0) break;
10659
+ for (const docId of ids) later.add(docId);
10660
+ for (const docId of tombstones) later.add(docId);
10661
+ }
10662
+ return replaced;
10663
+ }
10664
+ function exceedsRemovedShare(removed, held) {
10665
+ return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
10666
+ }
10667
+ function freedBytesOf(ref, replacedCount) {
10668
+ return ref.docCount === 0 ? 0 : ref.bytes * replacedCount / ref.docCount;
10669
+ }
10670
+ function chooseRewrites(segments, replaced, budget) {
10671
+ let held = 0;
10672
+ let removed = 0;
10673
+ for (let at = 0; at < segments.length; at++) {
10674
+ held += segments[at].docCount;
10675
+ removed += replaced[at].length;
10676
+ }
10677
+ const candidates = [];
10678
+ for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
10679
+ candidates.sort(
10680
+ (left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
10681
+ );
10682
+ const chosen = [];
10683
+ for (const at of candidates) {
10684
+ if (!exceedsRemovedShare(removed, held)) break;
10685
+ const bytes = segments[at].bytes;
10686
+ if (chosen.length > 0 && budget.spentBytes + bytes > budget.limitBytes) continue;
10687
+ chosen.push(at);
10688
+ budget.spentBytes += bytes;
10689
+ held -= replaced[at].length;
10690
+ removed -= replaced[at].length;
10691
+ }
10692
+ return chosen.sort((left, right) => left - right);
10693
+ }
10694
+ function holdsSomething(ref) {
10695
+ return ref.docCount > 0 || ref.tombstoneCount > 0;
10696
+ }
10697
+ async function rewriteRemovedDocuments(input) {
10698
+ const { segments, liveDocumentCount } = input;
10699
+ const unchanged = { segments, nextSegmentId: input.nextSegmentId };
10700
+ if (liveDocumentCount === void 0) return unchanged;
10701
+ let held = 0;
10702
+ for (const ref of segments) held += ref.docCount;
10703
+ if (!exceedsRemovedShare(held - liveDocumentCount, held)) return unchanged;
10704
+ const replaced = await documentsReplacedLater(input.store, segments);
10705
+ const chosen = new Set(chooseRewrites(segments, replaced, input.budget));
10706
+ if (chosen.size === 0) return unchanged;
10707
+ const rewritten = [];
10708
+ let nextSegmentId = input.nextSegmentId;
10709
+ for (let at = 0; at < segments.length; at++) {
10710
+ if (!chosen.has(at)) {
10711
+ rewritten.push(segments[at]);
10712
+ continue;
10713
+ }
10714
+ const written = await mergeSegmentWindow(
10715
+ { ...input, nextSegmentId },
10716
+ { refs: [segments[at]], startsTheList: rewritten.length === 0, replacedAfterWindow: replaced[at] }
10717
+ );
10718
+ if (written === null) {
10719
+ rewritten.push(segments[at]);
10720
+ continue;
10721
+ }
10722
+ nextSegmentId = written.nextSegmentId;
10723
+ rewritten.push(...written.segments.filter(holdsSomething));
10724
+ }
10725
+ return { segments: rewritten, nextSegmentId };
10726
+ }
10727
+
10500
10728
  // src/persistence/durability/segment/write.ts
10501
10729
  function textAnalysisOf(config) {
10502
10730
  let resolved = null;
@@ -10619,8 +10847,21 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
10619
10847
  compactionThreshold: context.compactionThreshold,
10620
10848
  paged: context.input.metadata.textStorage === "disk"
10621
10849
  });
10622
- segments = compacted.segments;
10623
- nextSegmentId = compacted.nextSegmentId;
10850
+ const rewritten = await rewriteRemovedDocuments({
10851
+ store: context.input.store,
10852
+ indexName: context.indexName,
10853
+ partitionId,
10854
+ schema: context.input.metadata.schema,
10855
+ partialWordAnalysis: context.partialWordAnalysis,
10856
+ textAnalysis: context.textAnalysis,
10857
+ segments: compacted.segments,
10858
+ nextSegmentId: compacted.nextSegmentId,
10859
+ paged: context.input.metadata.textStorage === "disk",
10860
+ liveDocumentCount: context.input.liveDocuments?.find((entry) => entry.partitionId === partitionId)?.documentCount,
10861
+ budget: context.rewriteBudget
10862
+ });
10863
+ segments = rewritten.segments;
10864
+ nextSegmentId = rewritten.nextSegmentId;
10624
10865
  return { partitionId, nextSegmentId, segments };
10625
10866
  }
10626
10867
  function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
@@ -10645,7 +10886,11 @@ async function writeCheckpointSegments(input) {
10645
10886
  partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
10646
10887
  textAnalysis,
10647
10888
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
10648
- compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
10889
+ compactionThreshold: resolveCompactionThreshold(input.compactionThreshold),
10890
+ rewriteBudget: {
10891
+ spentBytes: 0,
10892
+ limitBytes: input.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT
10893
+ }
10649
10894
  };
10650
10895
  const checkpointByPartition = /* @__PURE__ */ new Map();
10651
10896
  for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
@@ -10682,6 +10927,7 @@ async function writeCheckpointSegments(input) {
10682
10927
  async function commitCheckpointManifest(store, metadata, written, vectors) {
10683
10928
  const indexName = metadata.indexName;
10684
10929
  const manifest = {
10930
+ version: SEGMENT_MANIFEST_VERSION,
10685
10931
  schema: metadata.schema,
10686
10932
  language: metadata.language,
10687
10933
  ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
@@ -10691,9 +10937,9 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
10691
10937
  };
10692
10938
  const packed = await encodeSegmentManifest(manifest);
10693
10939
  await store.write(manifestKey(indexName), packed.chunks);
10694
- return { garbage: await unreferencedKeys(store, indexName, manifest) };
10940
+ return { manifest, saved: savedBytesOfManifest(manifest) };
10695
10941
  }
10696
- async function unreferencedKeys(store, indexName, manifest) {
10942
+ async function checkpointGarbage(store, indexName, manifest) {
10697
10943
  const referenced = manifestReferencedKeys(manifest);
10698
10944
  const stored = await store.list(segmentsPrefix(indexName));
10699
10945
  return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
@@ -10715,7 +10961,8 @@ function workerCanWriteEverySegment(input) {
10715
10961
  }
10716
10962
  async function writeIndexCheckpointSegments(input) {
10717
10963
  const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
10718
- const { keepsWriteRecords } = input;
10964
+ const { keepsWriteRecords, liveDocuments, rewriteBytesPerCheckpoint } = input;
10965
+ const counted = liveDocuments === null ? {} : { liveDocuments, rewriteBytesPerCheckpoint };
10719
10966
  const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
10720
10967
  root: directory.root,
10721
10968
  metadata,
@@ -10723,7 +10970,8 @@ async function writeIndexCheckpointSegments(input) {
10723
10970
  compactionThreshold,
10724
10971
  prior,
10725
10972
  capturedPartitions: [...capturedPartitions],
10726
- keepsWriteRecords
10973
+ keepsWriteRecords,
10974
+ ...counted
10727
10975
  }) : null;
10728
10976
  if (offloaded !== null) {
10729
10977
  return offloaded;
@@ -10737,7 +10985,8 @@ async function writeIndexCheckpointSegments(input) {
10737
10985
  prior,
10738
10986
  wholePartitions,
10739
10987
  capturedPartitions,
10740
- keepsWriteRecords
10988
+ keepsWriteRecords,
10989
+ ...counted
10741
10990
  });
10742
10991
  }
10743
10992
 
@@ -10820,61 +11069,6 @@ async function startReadingOpeningSectionsAhead(directory, manifest) {
10820
11069
  }
10821
11070
  }
10822
11071
 
10823
- // src/persistence/paged/segment-scan.ts
10824
- async function openPagedSegmentScan(directory, ref) {
10825
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
10826
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10827
- const segment = createPagedFrozenSegment(snapshot, cache);
10828
- const { paged } = segment;
10829
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10830
- const ids = segment.arrays.idTable;
10831
- const tombstones = new Set(snapshot.tombstones);
10832
- return {
10833
- count: snapshot.documentCount,
10834
- schema: snapshot.schema,
10835
- tombstones,
10836
- idAt: (ordinal) => ids.idAt(ordinal),
10837
- holds: (docId) => ids.ordinalOf(docId) >= 0,
10838
- storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
10839
- close: () => {
10840
- paged.close();
10841
- cache.clear();
10842
- }
10843
- };
10844
- }
10845
- async function checkPagedSegmentFully(directory, ref) {
10846
- const listing = await pagedListingOf(directory, ref);
10847
- await requirePagedPayloadChecksum(directory, listing);
10848
- const snapshot = openPagedSnapshot(listing);
10849
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10850
- const segment = createPagedFrozenSegment(snapshot, cache);
10851
- const { paged } = segment;
10852
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10853
- try {
10854
- await checkPagedContents(segment, cache);
10855
- return snapshot.schema;
10856
- } finally {
10857
- paged.close();
10858
- cache.clear();
10859
- }
10860
- }
10861
- async function bytesThatSegmentsHoldInMemory(directory, manifest) {
10862
- let bytes = 0;
10863
- for (const partition of manifest.partitions) {
10864
- for (const ref of partition.segments) {
10865
- const paged = await isPagedSegmentFile(directory, ref.key);
10866
- bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
10867
- }
10868
- }
10869
- return bytes;
10870
- }
10871
- function countsAsLatestHolder(scans, at, docId) {
10872
- for (let later = at + 1; later < scans.length; later++) {
10873
- if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
10874
- }
10875
- return true;
10876
- }
10877
-
10878
11072
  // src/persistence/durability/segment/load-partition.ts
10879
11073
  function requireAgreedSchema(agreed, schema, key) {
10880
11074
  for (const [field, type] of Object.entries(schema)) {
@@ -10888,6 +11082,20 @@ function requireAgreedSchema(agreed, schema, key) {
10888
11082
  }
10889
11083
  }
10890
11084
  }
11085
+ function requireListedCounts(ref, documentCount, tombstoneCount) {
11086
+ if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
11087
+ throw new NarsilError(
11088
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
11089
+ `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
11090
+ {
11091
+ key: ref.key,
11092
+ documentCount,
11093
+ tombstoneCount,
11094
+ listedDocCount: ref.docCount,
11095
+ listedTombstoneCount: ref.tombstoneCount
11096
+ }
11097
+ );
11098
+ }
10891
11099
  function removeWhatTheSegmentReplaces(manager, segment, contents) {
10892
11100
  manager.beginBatchRemove();
10893
11101
  try {
@@ -10922,6 +11130,7 @@ function removeFromPartition(manager, partitionId, docIds) {
10922
11130
  async function attachPagedSegment(directory, ref, partitionId, manager, options) {
10923
11131
  if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
10924
11132
  const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
11133
+ requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
10925
11134
  requireAgreedSchema(options.schema, snapshot.schema, ref.key);
10926
11135
  const segment = createPagedFrozenSegment(snapshot);
10927
11136
  removeFromPartition(manager, partitionId, snapshot.tombstones);
@@ -10967,6 +11176,7 @@ async function attachPartitionSegments(store, partition, manager, options) {
10967
11176
  }
10968
11177
  }
10969
11178
  const contents = await readSegmentFile(store, ref);
11179
+ requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
10970
11180
  requireAgreedSchema(options.schema, contents.schema, ref.key);
10971
11181
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
10972
11182
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
@@ -11425,23 +11635,28 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11425
11635
  metadata,
11426
11636
  targets,
11427
11637
  compactionThreshold: input.compactionThreshold,
11638
+ rewriteBytesPerCheckpoint: input.rewriteBytesPerCheckpoint,
11428
11639
  canOffload: input.canOffload,
11429
11640
  prior: priorForLayout(priorManifest, capture3.partitionCount),
11430
11641
  wholePartitions: capture3.wholePartitions.contents,
11431
11642
  capturedPartitions: capture3.wholePartitions.captured,
11432
- keepsWriteRecords: writeRecords !== null
11643
+ keepsWriteRecords: writeRecords !== null,
11644
+ liveDocuments: capture3.liveDocuments
11433
11645
  })
11434
11646
  );
11435
11647
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
11436
11648
  if (aTargetFailed(input.indexState, targets)) return 0;
11437
11649
  const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
11650
+ input.recordSavedBytes(written.saved);
11438
11651
  input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
11439
11652
  recordWrittenVectors(vectorIndexes, liveVectors.written);
11440
11653
  await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
11441
11654
  const changes = changesSinceCapture();
11442
11655
  const droppedThisCheckpoint = changes !== null && capture3.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
11656
+ await writeCheckpointMetadata(input, documentCount, capture3.partitionCount, written.saved.bytes);
11443
11657
  if (!input.holdsCheckpointFiles()) {
11444
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
11658
+ const garbage = await checkpointGarbage(store, input.indexName, written.manifest);
11659
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, garbage);
11445
11660
  const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
11446
11661
  const keptForSync = await input.keepsCheckpointKey();
11447
11662
  await removeCheckpointGarbage(
@@ -11449,19 +11664,6 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11449
11664
  removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
11450
11665
  );
11451
11666
  }
11452
- const checkpointDocumentCount = documentCount;
11453
- await input.queueMetadataWrite(input.indexName, async () => {
11454
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
11455
- if (checkpointMetadata === void 0) {
11456
- return;
11457
- }
11458
- const bytes = await writeMetadataEnvelope(
11459
- { ...checkpointMetadata, partitionCount: capture3.partitionCount },
11460
- { checksum: true }
11461
- );
11462
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
11463
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture3.partitionCount);
11464
- });
11465
11667
  const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
11466
11668
  await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
11467
11669
  if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
@@ -11479,6 +11681,18 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11479
11681
  const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
11480
11682
  return anotherIsDue ? eligibleBeyond : capture3.recordsLeftInLog;
11481
11683
  }
11684
+ async function writeCheckpointMetadata(input, documentCount, partitionCount, savedBytes) {
11685
+ const { indexName, hooks } = input;
11686
+ await input.queueMetadataWrite(indexName, async () => {
11687
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
11688
+ if (checkpointMetadata === void 0) {
11689
+ return;
11690
+ }
11691
+ const bytes = await writeMetadataEnvelope({ ...checkpointMetadata, partitionCount }, { checksum: true });
11692
+ await input.directory.atomicWrite(`${indexName}/meta`, bytes);
11693
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, savedBytes);
11694
+ });
11695
+ }
11482
11696
  function recordsBeyond(indexState, targets, cutoffs) {
11483
11697
  let records = 0;
11484
11698
  for (const [partitionId, partition] of indexState.partitions) {
@@ -11643,6 +11857,258 @@ function createHeldCheckpoints(hooks) {
11643
11857
  };
11644
11858
  }
11645
11859
 
11860
+ // src/persistence/durability/index-stored-sizes.ts
11861
+ function refreshStoredSizes(sizes) {
11862
+ const known = sizes.saved !== null || !sizes.measuring && sizes.held;
11863
+ sizes.known = known;
11864
+ if (known) {
11865
+ sizes.savedBytes = sizes.saved?.bytes ?? 0;
11866
+ sizes.vectorSavedBytes = sizes.saved?.vectorBytes ?? 0;
11867
+ sizes.logBytes = sizes.tracksLog ? sizes.logged?.bytes ?? 0 : void 0;
11868
+ return;
11869
+ }
11870
+ sizes.savedBytes = sizes.measuring ? void 0 : sizes.recorded;
11871
+ sizes.vectorSavedBytes = void 0;
11872
+ sizes.logBytes = void 0;
11873
+ }
11874
+ function savedBytesByPartitionOf(saved) {
11875
+ const byPartition = [];
11876
+ if (saved === null) return byPartition;
11877
+ for (const { partitionId, bytes } of saved.partitions) byPartition[partitionId] = bytes;
11878
+ return byPartition;
11879
+ }
11880
+ function createTrackedSizes(tracksLog) {
11881
+ const sizes = {
11882
+ savedBytes: void 0,
11883
+ logBytes: void 0,
11884
+ vectorSavedBytes: void 0,
11885
+ saved: null,
11886
+ savedBytesByPartition: [],
11887
+ measuring: false,
11888
+ held: false,
11889
+ recorded: void 0,
11890
+ logged: null,
11891
+ known: false,
11892
+ tracksLog,
11893
+ addTo(stats) {
11894
+ if (!sizes.known) return;
11895
+ const saved = sizes.savedBytesByPartition;
11896
+ const logged = sizes.tracksLog ? sizes.logged?.bytesByPartition ?? [] : null;
11897
+ for (const partition of stats) {
11898
+ partition.savedBytes = saved[partition.partitionId] ?? 0;
11899
+ if (logged !== null) partition.logBytes = logged[partition.partitionId] ?? 0;
11900
+ }
11901
+ },
11902
+ recordSavedBytes(bytes) {
11903
+ sizes.recorded = bytes;
11904
+ refreshStoredSizes(sizes);
11905
+ }
11906
+ };
11907
+ return sizes;
11908
+ }
11909
+ function createStoredSizesRecord(tracksLog) {
11910
+ const sizesByIndex = /* @__PURE__ */ new Map();
11911
+ function of(indexName) {
11912
+ let sizes = sizesByIndex.get(indexName);
11913
+ if (sizes === void 0) {
11914
+ sizes = createTrackedSizes(tracksLog);
11915
+ sizesByIndex.set(indexName, sizes);
11916
+ }
11917
+ return sizes;
11918
+ }
11919
+ function forget(indexName) {
11920
+ const sizes = sizesByIndex.get(indexName);
11921
+ if (sizes === void 0) return;
11922
+ sizes.saved = null;
11923
+ sizes.savedBytesByPartition = [];
11924
+ sizes.measuring = false;
11925
+ sizes.held = false;
11926
+ refreshStoredSizes(sizes);
11927
+ }
11928
+ return {
11929
+ of,
11930
+ setSaved(indexName, saved) {
11931
+ const sizes = of(indexName);
11932
+ sizes.saved = saved;
11933
+ sizes.savedBytesByPartition = savedBytesByPartitionOf(saved);
11934
+ refreshStoredSizes(sizes);
11935
+ },
11936
+ setMeasuring(indexName, measuring) {
11937
+ const sizes = of(indexName);
11938
+ sizes.measuring = measuring;
11939
+ refreshStoredSizes(sizes);
11940
+ },
11941
+ hold(indexName) {
11942
+ const sizes = of(indexName);
11943
+ if (!sizes.held) {
11944
+ sizes.held = true;
11945
+ refreshStoredSizes(sizes);
11946
+ }
11947
+ return sizes;
11948
+ },
11949
+ forget,
11950
+ drop(indexName) {
11951
+ forget(indexName);
11952
+ sizesByIndex.delete(indexName);
11953
+ }
11954
+ };
11955
+ }
11956
+
11957
+ // src/persistence/durability/log-bytes.ts
11958
+ var LOG_SEGMENT_KEY = /^(.+)\/wal\/(\d+)\/\d{16}$/;
11959
+ var HISTORY_SEGMENT_KEY = /^(.+)\/history\/(\d+)\/\d{16}-\d{16}$/;
11960
+ function loggedFileOf(key) {
11961
+ const match = LOG_SEGMENT_KEY.exec(key) ?? HISTORY_SEGMENT_KEY.exec(key);
11962
+ if (match === null) return null;
11963
+ const partitionId = Number.parseInt(match[2], 10);
11964
+ return Number.isSafeInteger(partitionId) ? { indexName: match[1], partitionId } : null;
11965
+ }
11966
+ function emptyLog() {
11967
+ return { files: /* @__PURE__ */ new Map(), bytes: 0, bytesByPartition: [], sizes: null };
11968
+ }
11969
+ function addBytes(log, partitionId, delta) {
11970
+ log.bytes += delta;
11971
+ log.bytesByPartition[partitionId] = (log.bytesByPartition[partitionId] ?? 0) + delta;
11972
+ if (log.sizes !== null) refreshStoredSizes(log.sizes);
11973
+ }
11974
+ function makeCurrent(previous, log, sizes) {
11975
+ if (previous !== void 0 && previous !== log) previous.sizes = null;
11976
+ log.sizes = sizes;
11977
+ sizes.logged = log;
11978
+ refreshStoredSizes(sizes);
11979
+ }
11980
+ function setFileBytes(log, key, partitionId, bytes) {
11981
+ const previous = log.files.get(key);
11982
+ if (previous !== void 0) addBytes(log, previous.partitionId, -previous.bytes);
11983
+ log.files.set(key, { partitionId, bytes });
11984
+ addBytes(log, partitionId, bytes);
11985
+ }
11986
+ function removeFile(log, key) {
11987
+ const previous = log.files.get(key);
11988
+ if (previous === void 0) return void 0;
11989
+ log.files.delete(key);
11990
+ addBytes(log, previous.partitionId, -previous.bytes);
11991
+ return previous;
11992
+ }
11993
+ function createLogBytesLedger(base, sizes) {
11994
+ const logs = /* @__PURE__ */ new Map();
11995
+ function logOf(indexName) {
11996
+ let log = logs.get(indexName);
11997
+ if (log === void 0) {
11998
+ log = emptyLog();
11999
+ logs.set(indexName, log);
12000
+ makeCurrent(void 0, log, sizes.of(indexName));
12001
+ }
12002
+ return log;
12003
+ }
12004
+ function track(key, bytes) {
12005
+ const file = loggedFileOf(key);
12006
+ if (file !== null) setFileBytes(logOf(file.indexName), key, file.partitionId, bytes);
12007
+ }
12008
+ function untrack(key) {
12009
+ const file = loggedFileOf(key);
12010
+ if (file === null) return void 0;
12011
+ const log = logs.get(file.indexName);
12012
+ return log === void 0 ? void 0 : removeFile(log, key);
12013
+ }
12014
+ async function currentLength(key) {
12015
+ return await base.byteLength?.(key) ?? (await base.read(key))?.byteLength ?? 0;
12016
+ }
12017
+ function trackedHandle(key, file, handle) {
12018
+ async function remeasure() {
12019
+ const log = logs.get(file.indexName);
12020
+ if (log?.files.has(key) === true) setFileBytes(log, key, file.partitionId, await handle.size());
12021
+ }
12022
+ return {
12023
+ async append(bytes) {
12024
+ try {
12025
+ await handle.append(bytes);
12026
+ } catch (error) {
12027
+ await remeasure().catch(() => void 0);
12028
+ throw error;
12029
+ }
12030
+ const log = logs.get(file.indexName);
12031
+ const entry = log?.files.get(key);
12032
+ if (log === void 0 || entry === void 0) return;
12033
+ entry.bytes += bytes.byteLength;
12034
+ addBytes(log, file.partitionId, bytes.byteLength);
12035
+ },
12036
+ sync: () => handle.sync(),
12037
+ async truncate(byteLength2) {
12038
+ try {
12039
+ await handle.truncate(byteLength2);
12040
+ } finally {
12041
+ await remeasure().catch(() => void 0);
12042
+ }
12043
+ },
12044
+ size: () => handle.size(),
12045
+ close: () => handle.close()
12046
+ };
12047
+ }
12048
+ const directory = {
12049
+ root: base.root,
12050
+ async appendHandle(key) {
12051
+ const handle = await base.appendHandle(key);
12052
+ const file = loggedFileOf(key);
12053
+ if (file === null) return handle;
12054
+ if (logs.get(file.indexName)?.files.has(key) !== true) track(key, await handle.size());
12055
+ return trackedHandle(key, file, handle);
12056
+ },
12057
+ markerHandle: (key) => base.markerHandle(key),
12058
+ async atomicWrite(key, data) {
12059
+ await base.atomicWrite(key, data);
12060
+ if (loggedFileOf(key) === null) return;
12061
+ let bytes = 0;
12062
+ for (const chunk of Array.isArray(data) ? data : [data]) bytes += chunk.byteLength;
12063
+ track(key, bytes);
12064
+ },
12065
+ syncDirectoryOf: (key) => base.syncDirectoryOf(key),
12066
+ read: (key) => base.read(key),
12067
+ async remove(key) {
12068
+ await base.remove(key);
12069
+ untrack(key);
12070
+ },
12071
+ async move(from, to) {
12072
+ await base.move(from, to);
12073
+ const moved = untrack(from);
12074
+ if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
12075
+ },
12076
+ list: (prefix) => base.list(prefix),
12077
+ pathOf: (key) => base.pathOf(key)
12078
+ };
12079
+ const { readRange, byteLength, identity, lock } = base;
12080
+ if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
12081
+ if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
12082
+ if (identity !== void 0) directory.identity = () => identity.call(base);
12083
+ if (lock !== void 0) directory.lock = () => lock.call(base);
12084
+ return {
12085
+ directory,
12086
+ async seed(indexName) {
12087
+ const log = emptyLog();
12088
+ for (const prefix of [`${indexName}/wal/`, `${indexName}/history/`]) {
12089
+ for (const key of await base.list(prefix)) {
12090
+ const file = loggedFileOf(key);
12091
+ if (file === null || file.indexName !== indexName) continue;
12092
+ setFileBytes(log, key, file.partitionId, await currentLength(key));
12093
+ }
12094
+ }
12095
+ const previous = logs.get(indexName);
12096
+ logs.set(indexName, log);
12097
+ makeCurrent(previous, log, sizes.of(indexName));
12098
+ },
12099
+ forget(indexName) {
12100
+ const log = logs.get(indexName);
12101
+ if (log === void 0) return;
12102
+ logs.delete(indexName);
12103
+ if (log.sizes !== null) {
12104
+ log.sizes.logged = null;
12105
+ refreshStoredSizes(log.sizes);
12106
+ log.sizes = null;
12107
+ }
12108
+ }
12109
+ };
12110
+ }
12111
+
11646
12112
  // src/persistence/durability/manager-state.ts
11647
12113
  function closePartitionWriter(partition) {
11648
12114
  return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
@@ -12057,6 +12523,7 @@ function createIndexUnloading(deps) {
12057
12523
  }
12058
12524
  await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
12059
12525
  deps.forgetSyncState(indexName);
12526
+ deps.dropStoredBytes(indexName);
12060
12527
  },
12061
12528
  async unloadIndex(indexName) {
12062
12529
  const indexState = indexes.get(indexName);
@@ -12067,6 +12534,7 @@ function createIndexUnloading(deps) {
12067
12534
  indexes.delete(indexName);
12068
12535
  metadataWrites.forget(indexName);
12069
12536
  deps.forgetSyncState(indexName);
12537
+ deps.forgetStoredBytes(indexName);
12070
12538
  }
12071
12539
  };
12072
12540
  }
@@ -12534,7 +13002,7 @@ function createSyncStorage(deps) {
12534
13002
 
12535
13003
  // src/persistence/durability/manager-storage.ts
12536
13004
  function createIndexStorageAccess(deps) {
12537
- const { directory, indexes } = deps;
13005
+ const { directory, logBytes, sizes } = deps;
12538
13006
  const archiveAccess = createArchiveCheckpointAccess({
12539
13007
  directory,
12540
13008
  held: deps.heldCheckpointFiles,
@@ -12546,7 +13014,7 @@ function createIndexStorageAccess(deps) {
12546
13014
  directory,
12547
13015
  openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
12548
13016
  inCheckpointTurn: (indexName, work) => {
12549
- const indexState = indexes.get(indexName);
13017
+ const indexState = deps.indexes.get(indexName);
12550
13018
  if (indexState === void 0) return work();
12551
13019
  return checkpointInTurn(indexState, {
12552
13020
  joinsInFlight: false,
@@ -12559,12 +13027,54 @@ function createIndexStorageAccess(deps) {
12559
13027
  const unloading = createIndexUnloading({
12560
13028
  directory,
12561
13029
  hooks: deps.hooks,
12562
- indexes,
13030
+ indexes: deps.indexes,
12563
13031
  metadataWrites: deps.metadataWrites,
12564
13032
  closeWriterReportingFailure: deps.closeWriterReportingFailure,
12565
- forgetSyncState: syncStorage.forget
13033
+ forgetSyncState: syncStorage.forget,
13034
+ forgetStoredBytes: (indexName) => {
13035
+ sizes.forget(indexName);
13036
+ logBytes.forget(indexName);
13037
+ },
13038
+ dropStoredBytes: (indexName) => {
13039
+ sizes.drop(indexName);
13040
+ logBytes.forget(indexName);
13041
+ }
12566
13042
  });
12567
- return { archiveAccess, syncStorage, unloading };
13043
+ async function measureStoredBytes(indexName, saved) {
13044
+ sizes.setSaved(indexName, saved);
13045
+ await logBytes.seed(indexName);
13046
+ }
13047
+ const recoveryDeps = {
13048
+ directory,
13049
+ hooks: deps.hooks,
13050
+ compactionThreshold: deps.compactionThreshold,
13051
+ registerPartition: deps.registerPartition,
13052
+ holdsCheckpointFiles: (indexName) => deps.heldCheckpointFiles.has(indexName),
13053
+ queueMetadataWrite: deps.metadataWrites.queue,
13054
+ syncStorage,
13055
+ measureStoredBytes
13056
+ };
13057
+ const storedBytesAccess = {
13058
+ storedSizesOf: sizes.of,
13059
+ addStoredSizes(indexName, stats) {
13060
+ sizes.of(indexName).addTo(stats);
13061
+ },
13062
+ vectorSavedBytes(indexName) {
13063
+ return sizes.of(indexName).vectorSavedBytes;
13064
+ },
13065
+ async measureRestoredStorage(indexName, restored) {
13066
+ if (restored !== void 0) {
13067
+ await measureStoredBytes(indexName, restored);
13068
+ return;
13069
+ }
13070
+ const decoded = await readSegmentManifest(directoryCheckpointStore(directory), indexName);
13071
+ await measureStoredBytes(
13072
+ indexName,
13073
+ decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED
13074
+ );
13075
+ }
13076
+ };
13077
+ return { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps };
12568
13078
  }
12569
13079
 
12570
13080
  // src/persistence/durability/metadata-write-queue.ts
@@ -13090,7 +13600,7 @@ function createWalWriter(directory, config) {
13090
13600
  }
13091
13601
 
13092
13602
  // src/persistence/durability/partition-registry.ts
13093
- function createPartitionRegistry(directory, segmentMaxBytes) {
13603
+ function createPartitionRegistry(directory, segmentMaxBytes, onIndexHeld) {
13094
13604
  const indexes = /* @__PURE__ */ new Map();
13095
13605
  function indexState(indexName) {
13096
13606
  let state = indexes.get(indexName);
@@ -13104,6 +13614,7 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
13104
13614
  stalledWrites: []
13105
13615
  };
13106
13616
  indexes.set(indexName, state);
13617
+ onIndexHeld(indexName);
13107
13618
  }
13108
13619
  return state;
13109
13620
  }
@@ -13639,7 +14150,7 @@ async function readEarlierSegmentDocuments(store, key) {
13639
14150
  }
13640
14151
 
13641
14152
  // src/persistence/durability/checkpoint-count.ts
13642
- var decoder2 = new TextDecoder();
14153
+ var decoder3 = new TextDecoder();
13643
14154
  async function countPagedPartition(store, partition) {
13644
14155
  const { directory } = store;
13645
14156
  if (directory === void 0 || partition.segments.length === 0) return null;
@@ -13672,7 +14183,7 @@ async function countCurrentLayout(store, manifest) {
13672
14183
  for (const docId of contents.tombstones) live.delete(docId);
13673
14184
  const ids = contents.arrays.idTable;
13674
14185
  for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
13675
- live.add(decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
14186
+ live.add(decoder3.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
13676
14187
  }
13677
14188
  }
13678
14189
  total += live.size;
@@ -13881,19 +14392,18 @@ async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
13881
14392
  const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
13882
14393
  recordWrittenVectors(deps.vectorIndexes, vectors.written);
13883
14394
  await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
13884
- await removeCheckpointGarbage(
13885
- deps.store,
13886
- await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, committed.garbage)
13887
- );
14395
+ const garbage = await checkpointGarbage(deps.store, deps.indexName, committed.manifest);
14396
+ await removeCheckpointGarbage(deps.store, await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, garbage));
14397
+ return committed.saved;
13888
14398
  }
13889
14399
  async function upgradeEarlierLayout(deps, earlier) {
13890
14400
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
13891
14401
  try {
13892
14402
  const loaded = await loadEarlier(deps, earlier);
13893
14403
  const targets = await deps.replay(loaded.checkpoint);
13894
- await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
14404
+ const saved = await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
13895
14405
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
13896
- return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
14406
+ return { checkpoint: targets, documentCount: deps.manager.countDocuments(), saved };
13897
14407
  } catch (error) {
13898
14408
  const failure = error instanceof Error ? error : new Error(String(error));
13899
14409
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
@@ -13913,11 +14423,11 @@ async function recoverCheckpoint(deps) {
13913
14423
  const earlier = await earlierLayoutOf(deps, decoded);
13914
14424
  if (earlier !== null) {
13915
14425
  const upgraded = await upgradeEarlierLayout(deps, earlier);
13916
- return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
14426
+ return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount, saved: upgraded.saved };
13917
14427
  }
13918
14428
  if (decoded === null || decoded.kind !== "current") {
13919
14429
  if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
13920
- return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
14430
+ return { checkpoint: await deps.replay([]), upgradedDocumentCount: null, saved: NOTHING_SAVED };
13921
14431
  }
13922
14432
  await deps.store.remove(snapshotBundleKey(deps.indexName));
13923
14433
  if (!deps.keepsUnlistedFiles) {
@@ -13930,7 +14440,11 @@ async function recoverCheckpoint(deps) {
13930
14440
  vectorIndexes: deps.vectorIndexes,
13931
14441
  writeRecords: deps.writeRecords ?? null
13932
14442
  });
13933
- return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
14443
+ return {
14444
+ checkpoint: await deps.replay(checkpoint),
14445
+ upgradedDocumentCount: null,
14446
+ saved: savedBytesOfManifest(decoded.manifest)
14447
+ };
13934
14448
  }
13935
14449
 
13936
14450
  // src/persistence/durability/layout-transition/recovery.ts
@@ -14032,9 +14546,9 @@ async function settleStoredAnalysis(store, metadata, metadataOnly) {
14032
14546
  }
14033
14547
 
14034
14548
  // src/persistence/durability/recover-index.ts
14035
- async function writeCountedMetadata(deps, indexName, documentCount) {
14549
+ async function writeCountedMetadata(deps, indexName, documentCount, savedBytes) {
14036
14550
  await deps.queueMetadataWrite(indexName, async () => {
14037
- const metadata = deps.hooks.buildMetadata(indexName, documentCount);
14551
+ const metadata = deps.hooks.buildMetadata(indexName, documentCount, savedBytes);
14038
14552
  if (metadata === void 0) return;
14039
14553
  const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
14040
14554
  await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
@@ -14129,8 +14643,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
14129
14643
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
14130
14644
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
14131
14645
  }
14646
+ await deps.measureStoredBytes(indexName, recovered.saved);
14132
14647
  if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
14133
- await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
14648
+ await writeCountedMetadata(
14649
+ deps,
14650
+ indexName,
14651
+ recovered.upgradedDocumentCount ?? manager.countDocuments(),
14652
+ recovered.saved.bytes
14653
+ );
14134
14654
  }
14135
14655
  return layout.transition === null ? null : removeUnread;
14136
14656
  }
@@ -14146,15 +14666,22 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
14146
14666
 
14147
14667
  // src/persistence/durability/manager.ts
14148
14668
  function createDurabilityManager(config, hooks, directoryOverride) {
14149
- const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
14669
+ const sizes = createStoredSizesRecord(true);
14670
+ const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
14671
+ const { directory } = logBytes;
14150
14672
  const canOffloadCheckpoint = directoryOverride === void 0;
14151
14673
  const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
14152
14674
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
14153
14675
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
14154
14676
  const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
14677
+ const rewriteBytesPerCheckpoint = config.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT;
14155
14678
  const mode = config.mode ?? "sync";
14156
14679
  const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
14157
- const registry = createPartitionRegistry(directory, segmentMaxBytes);
14680
+ const registry = createPartitionRegistry(
14681
+ directory,
14682
+ segmentMaxBytes,
14683
+ (indexName) => hooks.linkStoredSizes(indexName, sizes.hold(indexName))
14684
+ );
14158
14685
  const indexes = registry.indexes;
14159
14686
  const getOrCreateIndexState = registry.indexState;
14160
14687
  const getOrCreatePartition = registry.partition;
@@ -14245,12 +14772,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14245
14772
  indexName,
14246
14773
  indexState,
14247
14774
  compactionThreshold,
14775
+ rewriteBytesPerCheckpoint,
14248
14776
  canOffload: canOffloadCheckpoint,
14249
14777
  fromMemory,
14250
14778
  mutationThreshold: checkpointMutationThreshold,
14251
14779
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
14252
14780
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
14253
14781
  keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
14782
+ recordSavedBytes: (saved) => sizes.setSaved(indexName, saved),
14254
14783
  queueMetadataWrite,
14255
14784
  markFatal,
14256
14785
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -14305,28 +14834,22 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14305
14834
  startAsyncFlushTimer();
14306
14835
  return outcomes;
14307
14836
  }
14308
- const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
14837
+ const { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps } = createIndexStorageAccess({
14309
14838
  directory,
14839
+ logBytes,
14840
+ sizes,
14310
14841
  hooks,
14311
14842
  indexes,
14312
14843
  metadataWrites,
14313
14844
  heldCheckpointFiles,
14314
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
14315
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
14316
- markFatal,
14317
- closeWriterReportingFailure
14318
- });
14319
- const recoveryDeps = {
14320
- directory,
14321
- hooks,
14322
14845
  compactionThreshold,
14846
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
14323
14847
  registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
14324
14848
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
14325
14849
  },
14326
- holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
14327
- queueMetadataWrite,
14328
- syncStorage
14329
- };
14850
+ markFatal,
14851
+ closeWriterReportingFailure
14852
+ });
14330
14853
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
14331
14854
  const recovered = recoverWithLayoutCheckpoint(
14332
14855
  recoveryDeps,
@@ -14423,6 +14946,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14423
14946
  syncStorage: () => syncStorage,
14424
14947
  ...queryHistoryAccess({ kind: "directory", directory }),
14425
14948
  ...archiveAccess,
14949
+ ...storedBytesAccess,
14426
14950
  async shutdown() {
14427
14951
  shuttingDown = true;
14428
14952
  stopTimers();
@@ -14441,6 +14965,31 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14441
14965
  };
14442
14966
  }
14443
14967
 
14968
+ // src/persistence/durability/snapshot-only/index-state.ts
14969
+ function createSnapshotIndexState() {
14970
+ return {
14971
+ mutationsSinceCheckpoint: 0,
14972
+ checkpointInFlight: null,
14973
+ applyChain: Promise.resolve(),
14974
+ unloading: false,
14975
+ appliedSeqNoByPartition: /* @__PURE__ */ new Map()
14976
+ };
14977
+ }
14978
+ function metadataKey(indexName) {
14979
+ return `${indexName}/meta`;
14980
+ }
14981
+ function noteSnapshotCommitted(state, positions, partitionCount) {
14982
+ let recordsLeft = 0;
14983
+ for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
14984
+ if (partitionId >= partitionCount) {
14985
+ state.appliedSeqNoByPartition.delete(partitionId);
14986
+ continue;
14987
+ }
14988
+ recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
14989
+ }
14990
+ state.mutationsSinceCheckpoint = recordsLeft;
14991
+ }
14992
+
14444
14993
  // src/persistence/durability/snapshot-only/checkpoint.ts
14445
14994
  function capture(input, prior, cutoffs) {
14446
14995
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
@@ -14499,10 +15048,21 @@ async function writeCaptured(input, store, prior, captured) {
14499
15048
  });
14500
15049
  const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
14501
15050
  recordWrittenVectors(input.vectorIndexes, vectors.written);
14502
- await removeCheckpointGarbage(
14503
- store,
14504
- await garbageThatNoFieldStillReads(store, input.vectorIndexes, committed.garbage)
14505
- );
15051
+ await input.onCommitted({
15052
+ saved: committed.saved,
15053
+ documentCount: captured.documentCount,
15054
+ partitionCount: captured.partitionCount
15055
+ });
15056
+ const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
15057
+ await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
15058
+ }
15059
+ async function saveCheckpointMetadata(adapter, hooks, indexName, committed) {
15060
+ const { saved, documentCount, partitionCount } = committed;
15061
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, saved.bytes);
15062
+ if (checkpointMetadata === void 0) return;
15063
+ checkpointMetadata.partitionCount = partitionCount;
15064
+ await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(checkpointMetadata, { checksum: true }));
15065
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, saved.bytes);
14506
15066
  }
14507
15067
  async function writeSnapshotOnlyCheckpoint(input) {
14508
15068
  const store = adapterCheckpointStore(input.adapter);
@@ -14513,41 +15073,12 @@ async function writeSnapshotOnlyCheckpoint(input) {
14513
15073
  });
14514
15074
  try {
14515
15075
  await writeCaptured(input, store, prior, captured);
14516
- return {
14517
- documentCount: captured.documentCount,
14518
- positions: captured.positions,
14519
- partitionCount: captured.partitionCount
14520
- };
15076
+ return { positions: captured.positions, partitionCount: captured.partitionCount };
14521
15077
  } finally {
14522
15078
  for (const plan of captured.plans.values()) plan.release();
14523
15079
  }
14524
15080
  }
14525
15081
 
14526
- // src/persistence/durability/snapshot-only/index-state.ts
14527
- function createSnapshotIndexState() {
14528
- return {
14529
- mutationsSinceCheckpoint: 0,
14530
- checkpointInFlight: null,
14531
- applyChain: Promise.resolve(),
14532
- unloading: false,
14533
- appliedSeqNoByPartition: /* @__PURE__ */ new Map()
14534
- };
14535
- }
14536
- function metadataKey(indexName) {
14537
- return `${indexName}/meta`;
14538
- }
14539
- function noteSnapshotCommitted(state, positions, partitionCount) {
14540
- let recordsLeft = 0;
14541
- for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
14542
- if (partitionId >= partitionCount) {
14543
- state.appliedSeqNoByPartition.delete(partitionId);
14544
- continue;
14545
- }
14546
- recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
14547
- }
14548
- state.mutationsSinceCheckpoint = recordsLeft;
14549
- }
14550
-
14551
15082
  // src/persistence/durability/snapshot-only/restores.ts
14552
15083
  function toError6(error) {
14553
15084
  return error instanceof Error ? error : new Error(String(error));
@@ -14585,15 +15116,67 @@ function createPendingRestores(onFatalError) {
14585
15116
  };
14586
15117
  }
14587
15118
 
15119
+ // src/persistence/durability/snapshot-only/saved-size.ts
15120
+ function createSnapshotSavedSize(adapter, settleRestore) {
15121
+ const sizes = createStoredSizesRecord(false);
15122
+ const generations = /* @__PURE__ */ new Map();
15123
+ const store = adapterCheckpointStore(adapter);
15124
+ function replace(indexName, saved) {
15125
+ const generation = (generations.get(indexName) ?? 0) + 1;
15126
+ generations.set(indexName, generation);
15127
+ sizes.setSaved(indexName, saved);
15128
+ sizes.setMeasuring(indexName, saved === null);
15129
+ return generation;
15130
+ }
15131
+ async function measureAfterRestore(indexName, generation) {
15132
+ await settleRestore(indexName);
15133
+ const decoded = await readSegmentManifest(store, indexName);
15134
+ if (generations.get(indexName) !== generation) return;
15135
+ replace(indexName, decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED);
15136
+ }
15137
+ return {
15138
+ record(indexName, saved) {
15139
+ replace(indexName, saved);
15140
+ },
15141
+ hold: sizes.hold,
15142
+ forget(indexName) {
15143
+ generations.delete(indexName);
15144
+ sizes.forget(indexName);
15145
+ },
15146
+ drop(indexName) {
15147
+ generations.delete(indexName);
15148
+ sizes.drop(indexName);
15149
+ },
15150
+ access: {
15151
+ storedSizesOf: sizes.of,
15152
+ addStoredSizes(indexName, stats) {
15153
+ sizes.of(indexName).addTo(stats);
15154
+ },
15155
+ vectorSavedBytes(indexName) {
15156
+ return sizes.of(indexName).vectorSavedBytes;
15157
+ },
15158
+ async measureRestoredStorage(indexName, restored) {
15159
+ if (restored !== void 0) {
15160
+ replace(indexName, restored);
15161
+ return;
15162
+ }
15163
+ const generation = replace(indexName, null);
15164
+ void measureAfterRestore(indexName, generation).catch(() => void 0);
15165
+ }
15166
+ }
15167
+ };
15168
+ }
15169
+
14588
15170
  // src/persistence/durability/snapshot-only/index.ts
14589
15171
  function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14590
15172
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
14591
15173
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
14592
15174
  const indexes = /* @__PURE__ */ new Map();
14593
- const metadataWrites = /* @__PURE__ */ new Map();
15175
+ const metadataWrites = createMetadataWriteQueue();
14594
15176
  const loads = createIndexLoads(hooks.getVectorIndexes);
14595
15177
  const restoreStorage = adapterRestoreStorage(adapter);
14596
15178
  const restores = createPendingRestores(hooks.onFatalError);
15179
+ const savedSize = createSnapshotSavedSize(adapter, settleRestore);
14597
15180
  let checkpointTimer = null;
14598
15181
  let shuttingDown = false;
14599
15182
  function settleRestore(indexName) {
@@ -14604,6 +15187,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14604
15187
  if (state === void 0) {
14605
15188
  state = createSnapshotIndexState();
14606
15189
  indexes.set(indexName, state);
15190
+ hooks.linkStoredSizes(indexName, savedSize.hold(indexName));
14607
15191
  }
14608
15192
  return state;
14609
15193
  }
@@ -14632,23 +15216,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14632
15216
  const manager = hooks.getManager(indexName);
14633
15217
  const metadata = hooks.buildTermsMetadata(indexName);
14634
15218
  if (manager === void 0 || metadata === void 0) return;
14635
- const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
15219
+ const { positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
14636
15220
  adapter,
14637
15221
  indexName,
14638
15222
  metadata,
14639
15223
  manager,
14640
15224
  vectorIndexes: hooks.getVectorIndexes(indexName),
14641
15225
  appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
14642
- rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
14643
- });
14644
- hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
14645
- await queueMetadataWrite(indexName, async () => {
14646
- const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
14647
- if (checkpointMetadata === void 0) return;
14648
- checkpointMetadata.partitionCount = partitionCount;
14649
- const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
14650
- await adapter.save(metadataKey(indexName), metadataBytes);
14651
- hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
15226
+ rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
15227
+ onCommitted: async (committed) => {
15228
+ savedSize.record(indexName, committed.saved);
15229
+ hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
15230
+ await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
15231
+ }
14652
15232
  });
14653
15233
  noteSnapshotCommitted(indexState, positions, partitionCount);
14654
15234
  if (publisher !== void 0) {
@@ -14674,16 +15254,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14674
15254
  );
14675
15255
  }
14676
15256
  function queueMetadataWrite(indexName, write) {
14677
- const previous = metadataWrites.get(indexName) ?? Promise.resolve();
14678
- const queued = previous.then(() => settleRestore(indexName)).then(write);
14679
- metadataWrites.set(
14680
- indexName,
14681
- queued.catch(() => void 0)
14682
- );
14683
- return queued;
15257
+ return metadataWrites.queue(indexName, () => settleRestore(indexName).then(write));
14684
15258
  }
14685
- async function saveMetadata(indexName, documentCount) {
14686
- const metadata = hooks.buildMetadata(indexName, documentCount);
15259
+ async function saveMetadata(indexName, documentCount, savedBytes) {
15260
+ const metadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
14687
15261
  if (metadata === void 0) return;
14688
15262
  await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
14689
15263
  }
@@ -14693,9 +15267,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14693
15267
  async function loadIndex(indexName, metadataOnly) {
14694
15268
  await settleRestore(indexName);
14695
15269
  const metaBytes = await adapter.load(metadataKey(indexName));
14696
- if (metaBytes === null) {
14697
- return;
14698
- }
15270
+ if (metaBytes === null) return;
14699
15271
  const { metadata } = await readMetadataEnvelope(metaBytes);
14700
15272
  const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
14701
15273
  let derivedCount;
@@ -14734,6 +15306,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14734
15306
  onLayoutUpgrade: hooks.onLayoutUpgrade
14735
15307
  });
14736
15308
  const indexState = getOrCreateIndexState(indexName);
15309
+ savedSize.record(indexName, recovered.saved);
14737
15310
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
14738
15311
  const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
14739
15312
  if (persistedSeqNo > 0) {
@@ -14742,7 +15315,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14742
15315
  }
14743
15316
  if (recovered.upgradedDocumentCount !== null || promoted) {
14744
15317
  const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
14745
- await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
15318
+ await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount, recovered.saved.bytes)).catch(
15319
+ () => void 0
15320
+ );
14746
15321
  }
14747
15322
  }
14748
15323
  async function recordMutation(record) {
@@ -14774,14 +15349,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14774
15349
  try {
14775
15350
  await indexState.applyChain;
14776
15351
  if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
14777
- await metadataWrites.get(indexName);
15352
+ await metadataWrites.pending(indexName);
14778
15353
  } catch (error) {
14779
15354
  indexState.unloading = false;
14780
15355
  throw error;
14781
15356
  }
14782
15357
  }
14783
15358
  indexes.delete(indexName);
14784
- metadataWrites.delete(indexName);
15359
+ metadataWrites.forget(indexName);
15360
+ savedSize.forget(indexName);
14785
15361
  }
14786
15362
  return {
14787
15363
  isActive() {
@@ -14863,6 +15439,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14863
15439
  }
14864
15440
  },
14865
15441
  ...queryHistoryAccess({ kind: "adapter", adapter }),
15442
+ ...savedSize.access,
14866
15443
  async removeIndex(indexName) {
14867
15444
  await settleRestore(indexName).catch(() => void 0);
14868
15445
  restores.forget(indexName);
@@ -14872,6 +15449,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14872
15449
  await adapter.delete(key);
14873
15450
  }
14874
15451
  await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
15452
+ savedSize.drop(indexName);
14875
15453
  },
14876
15454
  async unloadIndex(indexName) {
14877
15455
  await settleRestore(indexName);
@@ -14916,7 +15494,7 @@ function vectorFieldsOf(schema, promotion) {
14916
15494
  }
14917
15495
  return Object.keys(fields).length === 0 ? null : fields;
14918
15496
  }
14919
- function metadataFromConfig(indexName, config, partitionCount, documentCount) {
15497
+ function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
14920
15498
  const metadata = {
14921
15499
  indexName,
14922
15500
  schema: config.schema,
@@ -14931,6 +15509,8 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
14931
15509
  engineVersion: VERSION
14932
15510
  };
14933
15511
  metadata.documentCount = documentCount ?? config.documentCount;
15512
+ const recordedSavedBytes = savedBytes ?? config.savedBytes;
15513
+ if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
14934
15514
  const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
14935
15515
  if (vectorFields !== null) {
14936
15516
  metadata.vectorFields = vectorFields;
@@ -15010,6 +15590,7 @@ function metadataConfigOf(entry) {
15010
15590
  ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
15011
15591
  ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
15012
15592
  documentCount: entry.documentCount,
15593
+ savedBytes: entry.savedBytes,
15013
15594
  partitionCount: entry.partitionCount,
15014
15595
  schema: flattenSchema(entry.config.schema),
15015
15596
  language: entry.language.name,
@@ -15088,7 +15669,7 @@ function indexMetadataOf(indexName, entry, partitionCount) {
15088
15669
  }
15089
15670
 
15090
15671
  // src/engine/durability-integration.ts
15091
- function buildMetadata(indexName, hooks, documentCount) {
15672
+ function buildMetadata(indexName, hooks, documentCount, savedBytes) {
15092
15673
  const config = hooks.getIndexConfig(indexName);
15093
15674
  if (config === void 0) {
15094
15675
  return void 0;
@@ -15098,7 +15679,8 @@ function buildMetadata(indexName, hooks, documentCount) {
15098
15679
  indexName,
15099
15680
  config,
15100
15681
  manager === void 0 ? void 0 : metadataPartitionCount(manager),
15101
- documentCount
15682
+ documentCount,
15683
+ savedBytes
15102
15684
  );
15103
15685
  }
15104
15686
  function createDurabilityIntegration(tier, hooks) {
@@ -15106,7 +15688,7 @@ function createDurabilityIntegration(tier, hooks) {
15106
15688
  getManager: hooks.getManager,
15107
15689
  getVectorFieldPaths: hooks.getVectorFieldPaths,
15108
15690
  getVectorIndexes: hooks.getVectorIndexes,
15109
- buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
15691
+ buildMetadata: (indexName, documentCount, savedBytes) => buildMetadata(indexName, hooks, documentCount, savedBytes),
15110
15692
  buildTermsMetadata(indexName) {
15111
15693
  const config = hooks.getTermsConfig(indexName);
15112
15694
  return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
@@ -15121,7 +15703,8 @@ function createDurabilityIntegration(tier, hooks) {
15121
15703
  onCheckpointHeld: hooks.onCheckpointHeld,
15122
15704
  onFatalError: hooks.onFatalError,
15123
15705
  ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
15124
- ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
15706
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved },
15707
+ linkStoredSizes: hooks.linkStoredSizes
15125
15708
  };
15126
15709
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
15127
15710
  async function recordMutation(indexName, docId, operation, document, apply) {
@@ -15242,7 +15825,7 @@ var OLD_REGULAR_FILE = 0;
15242
15825
  var PAX_EXTENDED_HEADER = 120;
15243
15826
  var PAX_HEADER_NAME = "PaxHeader";
15244
15827
  var encoder3 = new TextEncoder();
15245
- var decoder3 = new TextDecoder("utf-8", { fatal: true });
15828
+ var decoder4 = new TextDecoder("utf-8", { fatal: true });
15246
15829
  function writeString(block, offset, length, value) {
15247
15830
  const bytes = encoder3.encode(value);
15248
15831
  block.set(bytes.subarray(0, length), offset);
@@ -15320,7 +15903,7 @@ function readString(block, offset, length) {
15320
15903
  let end = offset;
15321
15904
  while (end < offset + length && block[end] !== 0) end++;
15322
15905
  try {
15323
- return decoder3.decode(block.subarray(offset, end));
15906
+ return decoder4.decode(block.subarray(offset, end));
15324
15907
  } catch {
15325
15908
  invalidArchive("an entry header holds a name that is not UTF-8");
15326
15909
  }
@@ -15357,7 +15940,7 @@ var EQUALS = 61;
15357
15940
  var NEWLINE = 10;
15358
15941
  function decodeRecordText(bytes) {
15359
15942
  try {
15360
- return decoder3.decode(bytes);
15943
+ return decoder4.decode(bytes);
15361
15944
  } catch {
15362
15945
  invalidArchive("a pax extended header is not UTF-8");
15363
15946
  }
@@ -16045,7 +16628,7 @@ function createInvalidationIntegration(deps) {
16045
16628
 
16046
16629
  // src/engine/durability-wiring.ts
16047
16630
  function createDurabilityFromTier(tier, wiring) {
16048
- return createDurabilityIntegration(tier, {
16631
+ const integration = createDurabilityIntegration(tier, {
16049
16632
  checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
16050
16633
  getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
16051
16634
  getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
@@ -16065,8 +16648,25 @@ function createDurabilityFromTier(tier, wiring) {
16065
16648
  },
16066
16649
  queueDiskAdoption: wiring.queueDiskAdoption,
16067
16650
  awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
16068
- createIndexFromMetadata: wiring.createIndexFromMetadata,
16069
- recordCheckpoint: wiring.recordCheckpoint,
16651
+ async createIndexFromMetadata(metadata, loadData) {
16652
+ await wiring.createIndexFromMetadata(metadata, loadData);
16653
+ const entry = wiring.indexRegistry.get(metadata.indexName);
16654
+ if (entry === void 0) return;
16655
+ entry.storedSizes = integration.manager.storedSizesOf(metadata.indexName);
16656
+ entry.storedSizes.recordSavedBytes(entry.savedBytes);
16657
+ },
16658
+ recordCheckpoint(indexName, documentCount, partitionCount, savedBytes) {
16659
+ const entry = wiring.indexRegistry.get(indexName);
16660
+ if (entry === void 0) return;
16661
+ entry.documentCount = documentCount;
16662
+ entry.partitionCount = partitionCount;
16663
+ entry.savedBytes = savedBytes;
16664
+ entry.storedSizes?.recordSavedBytes(savedBytes);
16665
+ },
16666
+ linkStoredSizes(indexName, sizes) {
16667
+ const entry = wiring.indexRegistry.get(indexName);
16668
+ if (entry !== void 0) entry.storedSizes = sizes;
16669
+ },
16070
16670
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
16071
16671
  onCheckpointHeld: wiring.emitCheckpointHeld,
16072
16672
  onFatalError: wiring.emitFatalError,
@@ -16080,6 +16680,7 @@ function createDurabilityFromTier(tier, wiring) {
16080
16680
  }
16081
16681
  }
16082
16682
  });
16683
+ return integration;
16083
16684
  }
16084
16685
  async function reloadIndexAndCopies(orchestrator, durability, indexName) {
16085
16686
  const manager = durability?.manager;
@@ -16425,7 +17026,7 @@ function joined2(parts, total) {
16425
17026
  return bytes;
16426
17027
  }
16427
17028
  async function* exportLines(source, maxLineBytes) {
16428
- const decoder5 = new TextDecoder("utf-8", { fatal: true });
17029
+ const decoder6 = new TextDecoder("utf-8", { fatal: true });
16429
17030
  let parts = [];
16430
17031
  let pending = 0;
16431
17032
  let overflowing = false;
@@ -16442,7 +17043,7 @@ async function* exportLines(source, maxLineBytes) {
16442
17043
  parts = [];
16443
17044
  pending = 0;
16444
17045
  try {
16445
- return { line, text: decoder5.decode(bytes) };
17046
+ return { line, text: decoder6.decode(bytes) };
16446
17047
  } catch {
16447
17048
  return { line, refused: "not-utf8" };
16448
17049
  }
@@ -16977,10 +17578,10 @@ async function readSegment(checkpoint, ref) {
16977
17578
  });
16978
17579
  return decodeKeywordSegment(payload, false);
16979
17580
  }
16980
- var decoder4 = new TextDecoder();
17581
+ var decoder5 = new TextDecoder();
16981
17582
  function idAt(contents, ordinal) {
16982
17583
  const ids = contents.arrays.idTable;
16983
- return decoder4.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
17584
+ return decoder5.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
16984
17585
  }
16985
17586
  async function winnersOf(checkpoint, partition) {
16986
17587
  const winners = /* @__PURE__ */ new Map();
@@ -17501,8 +18102,9 @@ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
17501
18102
  partitionCount ?? void 0
17502
18103
  );
17503
18104
  }
17504
- function restoredRegistryEntry(prepared, manager, deps) {
18105
+ function restoredRegistryEntry(indexName, prepared, manager, deps) {
17505
18106
  return {
18107
+ storedSizes: deps.durability?.manager.storedSizesOf(indexName),
17506
18108
  config: prepared.config,
17507
18109
  language: prepared.language,
17508
18110
  embeddingAdapter: prepared.embeddingAdapter,
@@ -17602,13 +18204,17 @@ async function commitToStorage(indexName, restored, previous, deps) {
17602
18204
  for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
17603
18205
  await staging.commit();
17604
18206
  } catch (error) {
17605
- if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
18207
+ if (previousPartitions > 0) {
18208
+ durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
18209
+ await durable.measureRestoredStorage?.(indexName).catch(() => void 0);
18210
+ }
17606
18211
  throw error;
17607
18212
  }
17608
18213
  for (const paged of pagedSegmentsOf(previous)) paged.close();
17609
18214
  const restoredSegments = pagedSegmentsOf(restored.detached.manager);
17610
18215
  for (const paged of restoredSegments) paged.suspend();
17611
18216
  await staging.finish();
18217
+ await durable.measureRestoredStorage?.(indexName, restored.saved ?? void 0).catch(() => void 0);
17612
18218
  const relocate = await staging.relocation().catch(() => null);
17613
18219
  for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
17614
18220
  if (relocate === null) return;
@@ -17617,6 +18223,7 @@ async function commitToStorage(indexName, restored, previous, deps) {
17617
18223
  function activate(indexName, restored, entry, deps) {
17618
18224
  deps.executor.installDetachedIndex(restored.detached);
17619
18225
  deps.indexRegistry.set(indexName, entry);
18226
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
17620
18227
  restored.detached.attach();
17621
18228
  deps.clearAnalysisStale(indexName);
17622
18229
  const { prepared } = restored;
@@ -17656,6 +18263,7 @@ async function createFresh(indexName, restored, entry, deps) {
17656
18263
  deps.copies.desyncIndex(indexName);
17657
18264
  deps.executor.installDetachedIndex(restored.detached);
17658
18265
  deps.indexRegistry.set(indexName, entry);
18266
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
17659
18267
  try {
17660
18268
  await deps.indexState.registerReplacing(indexName, async () => {
17661
18269
  await commitToStorage(indexName, restored, void 0, deps);
@@ -17674,9 +18282,10 @@ async function stageMetadata(indexName, staging, entry) {
17674
18282
  await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
17675
18283
  }
17676
18284
  async function installRestoredIndex(indexName, restored, deps, history = null) {
17677
- const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
18285
+ const entry = restoredRegistryEntry(indexName, restored.prepared, restored.detached.manager, deps);
17678
18286
  const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
17679
18287
  if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
18288
+ if (restored.saved !== null) entry.savedBytes = restored.saved.bytes;
17680
18289
  let replaced;
17681
18290
  try {
17682
18291
  if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
@@ -17809,7 +18418,7 @@ async function loadStagedIndex(indexName, staged, staging, store, deps) {
17809
18418
  deps.executor.discardDetachedIndex(detached);
17810
18419
  throw error;
17811
18420
  }
17812
- return { prepared, detached, staging, checkpoint: manifest.checkpoint };
18421
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint, saved: savedBytesOfManifest(manifest) };
17813
18422
  }
17814
18423
  async function restoreArchive(indexName, reader, deps) {
17815
18424
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
@@ -17910,7 +18519,7 @@ function insertSnapshotDocuments(envelope, manager) {
17910
18519
  for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
17911
18520
  }
17912
18521
  async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
17913
- const entry = restoredRegistryEntry(prepared, built.manager, deps);
18522
+ const entry = restoredRegistryEntry(indexName, prepared, built.manager, deps);
17914
18523
  const archive = await buildArchiveFromMemory({
17915
18524
  indexName,
17916
18525
  metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
@@ -17934,7 +18543,7 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
17934
18543
  let manifest;
17935
18544
  try {
17936
18545
  insertSnapshotDocuments(envelope, built.manager);
17937
- if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
18546
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [], saved: null };
17938
18547
  manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
17939
18548
  } catch (err) {
17940
18549
  deps.executor.discardDetachedIndex(built);