@delali/narsil 0.3.1-next.181 → 0.3.1-next.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
  4. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
  5. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  6. package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
  7. package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
  8. package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
  9. package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
  10. package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
  11. package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
  12. package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
  13. package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
  14. package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
  15. package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
  16. package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
  17. package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
  18. package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
  19. package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
  20. package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
  21. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  22. package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
  23. package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
  24. package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
  25. package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
  26. package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
  27. package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
  28. package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
  29. package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
  30. package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
  31. package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
  32. package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
  33. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  34. package/dist/client.d.ts +6 -6
  35. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  36. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  37. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
  38. package/dist/distribution.d.ts +4 -4
  39. package/dist/distribution.mjs +391 -138
  40. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  41. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  42. package/dist/index.browser.mjs +1 -1
  43. package/dist/index.d.ts +4 -4
  44. package/dist/index.mjs +1 -1
  45. package/dist/index.react-native.mjs +1 -1
  46. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
  47. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
  48. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
  49. package/dist/partial-words.browser.mjs +1 -1
  50. package/dist/partial-words.mjs +1 -1
  51. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  52. package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
  53. package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
  54. package/dist/query-suggestions/merge-worker.mjs +4 -4
  55. package/dist/query-suggestions.browser.mjs +1 -1
  56. package/dist/query-suggestions.d.ts +1 -1
  57. package/dist/query-suggestions.mjs +2 -2
  58. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
  59. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
  60. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
  61. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
  62. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
  63. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
  64. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  65. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
  66. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
  67. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
  68. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
  69. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
  70. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
  71. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
  72. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
  73. package/dist/react-native/partial-words.mjs +1 -1
  74. package/dist/react-native/query-suggestions.mjs +1 -1
  75. package/dist/react-native/sync.mjs +1 -1
  76. package/dist/react-native/synonyms.mjs +1 -1
  77. package/dist/react.d.ts +7 -7
  78. package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
  79. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
  80. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
  81. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
  82. package/dist/server.d.ts +8 -8
  83. package/dist/server.mjs +16 -16
  84. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
  85. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
  86. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
  87. package/dist/sync/filesystem.d.ts +2 -2
  88. package/dist/sync/filesystem.mjs +6 -6
  89. package/dist/sync/indexeddb.d.ts +2 -2
  90. package/dist/sync.browser.mjs +1 -1
  91. package/dist/sync.d.ts +3 -3
  92. package/dist/sync.mjs +1 -1
  93. package/dist/synonyms.browser.mjs +1 -1
  94. package/dist/synonyms.mjs +1 -1
  95. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  96. package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
  97. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  98. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
  99. package/dist/workers/entry.mjs +7 -7
  100. package/openapi.json +32 -0
  101. package/package.json +7 -7
@@ -2109,6 +2109,8 @@ var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
2109
2109
  var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
2110
2110
  var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
2111
2111
  var LOG_RECORDS_PER_CHECKPOINT = 1e5;
2112
+ var REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE = 0.2;
2113
+ var DEFAULT_REWRITE_BYTES_PER_CHECKPOINT = 268435456;
2112
2114
  var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
2113
2115
  var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
2114
2116
  var CHECKPOINT_WORKER_IDLE_MS = 5e3;
@@ -2691,6 +2693,9 @@ function metadataToWire(meta) {
2691
2693
  if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
2692
2694
  wire.document_count = meta.documentCount;
2693
2695
  }
2696
+ if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
2697
+ wire.saved_bytes = meta.savedBytes;
2698
+ }
2694
2699
  if (meta.embedding) {
2695
2700
  wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
2696
2701
  }
@@ -2806,6 +2811,9 @@ function wireToMetadata(raw) {
2806
2811
  if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
2807
2812
  meta.documentCount = raw.document_count;
2808
2813
  }
2814
+ if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
2815
+ meta.savedBytes = raw.saved_bytes;
2816
+ }
2809
2817
  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
2810
2818
  meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
2811
2819
  }
@@ -4179,6 +4187,12 @@ function createReadCache(capacity) {
4179
4187
  function evictDownTo(limit) {
4180
4188
  for (const [key, entry] of entries) {
4181
4189
  if (held <= limit) return;
4190
+ if (entry.referenced) {
4191
+ entry.referenced = false;
4192
+ entries.delete(key);
4193
+ entries.set(key, entry);
4194
+ continue;
4195
+ }
4182
4196
  drop(key, entry);
4183
4197
  }
4184
4198
  }
@@ -4233,8 +4247,7 @@ function createReadCache(capacity) {
4233
4247
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
4234
4248
  const found = entries.get(key);
4235
4249
  if (found === void 0) return void 0;
4236
- entries.delete(key);
4237
- entries.set(key, found);
4250
+ found.referenced = true;
4238
4251
  return found.value;
4239
4252
  },
4240
4253
  put(fileNumber, kind, entry, value, bytes) {
@@ -4242,7 +4255,7 @@ function createReadCache(capacity) {
4242
4255
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
4243
4256
  const existing = entries.get(key);
4244
4257
  if (existing !== void 0) drop(key, existing);
4245
- entries.set(key, { value, bytes, fileNumber });
4258
+ entries.set(key, { value, bytes, fileNumber, referenced: true });
4246
4259
  held += bytes;
4247
4260
  perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
4248
4261
  evictDownTo(capacity - reserved);
@@ -5542,28 +5555,55 @@ async function pagedListingOf(directory, ref) {
5542
5555
  function yieldToEventLoop2() {
5543
5556
  return new Promise((resolve) => setTimeout(resolve, 0));
5544
5557
  }
5558
+ function newerTombstonesOf(prepared, at) {
5559
+ const docIds = /* @__PURE__ */ new Set();
5560
+ for (let newer = at + 1; newer < prepared.length; newer++) {
5561
+ const access = prepared[newer].segment.paged;
5562
+ if (access === null) continue;
5563
+ for (const docId of access.snapshot.tombstones) docIds.add(docId);
5564
+ }
5565
+ return docIds;
5566
+ }
5567
+ function scansEveryId(segment, tombstoned) {
5568
+ const held = segment.arrays.idTable.count;
5569
+ return tombstoned.size * Math.ceil(Math.log2(held + 1)) > held;
5570
+ }
5571
+ function tombstoneScannedOrdinal(segment, tombstoned, ordinal) {
5572
+ const docId = segment.arrays.idTable.idAt(ordinal);
5573
+ if (tombstoned.has(docId)) segment.prepareTombstone(docId, void 0, void 0, ordinal)?.publish();
5574
+ }
5545
5575
  function applyNewerTombstones(prepared) {
5546
5576
  for (let at = 0; at < prepared.length; at++) {
5547
5577
  if (!prepared[at].isNew) continue;
5548
- for (let newer = at + 1; newer < prepared.length; newer++) {
5549
- const access = prepared[newer].segment.paged;
5550
- if (access === null) continue;
5551
- for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
5578
+ const { segment } = prepared[at];
5579
+ const tombstoned = newerTombstonesOf(prepared, at);
5580
+ if (!scansEveryId(segment, tombstoned)) {
5581
+ for (const docId of tombstoned) segment.tombstoneDocument(docId);
5582
+ continue;
5552
5583
  }
5584
+ const held = segment.arrays.idTable.count;
5585
+ for (let ordinal = 0; ordinal < held; ordinal++) tombstoneScannedOrdinal(segment, tombstoned, ordinal);
5553
5586
  }
5554
5587
  }
5555
5588
  async function applyNewerTombstonesInTurns(prepared) {
5556
- let applied = 0;
5589
+ let steps = 0;
5557
5590
  for (let at = 0; at < prepared.length; at++) {
5558
5591
  if (!prepared[at].isNew) continue;
5559
- for (let newer = at + 1; newer < prepared.length; newer++) {
5560
- const access = prepared[newer].segment.paged;
5561
- if (access === null) continue;
5562
- for (const docId of access.snapshot.tombstones) {
5563
- prepared[at].segment.tombstoneDocument(docId);
5564
- applied += 1;
5565
- if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5592
+ const { segment } = prepared[at];
5593
+ const tombstoned = newerTombstonesOf(prepared, at);
5594
+ if (!scansEveryId(segment, tombstoned)) {
5595
+ for (const docId of tombstoned) {
5596
+ segment.tombstoneDocument(docId);
5597
+ steps += 1;
5598
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5566
5599
  }
5600
+ continue;
5601
+ }
5602
+ const held = segment.arrays.idTable.count;
5603
+ for (let ordinal = 0; ordinal < held; ordinal++) {
5604
+ tombstoneScannedOrdinal(segment, tombstoned, ordinal);
5605
+ steps += 1;
5606
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5567
5607
  }
5568
5608
  }
5569
5609
  }
@@ -8237,6 +8277,9 @@ function wholePartitionsWhereMostChanged(manager, rules) {
8237
8277
  return whole;
8238
8278
  };
8239
8279
  }
8280
+ function liveDocumentsOf(manager, targets) {
8281
+ return targets.map(({ partitionId }) => ({ partitionId, documentCount: manager.getPartition(partitionId).count() }));
8282
+ }
8240
8283
  async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
8241
8284
  const whileNoBuildPassOrMutationRuns = (capture3) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture3));
8242
8285
  return whileNoBuildPassOrMutationRuns(
@@ -8279,7 +8322,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
8279
8322
  vectorPlans,
8280
8323
  wholePartitions,
8281
8324
  recordsLeftInLog,
8282
- unreplayedCutoffs: cutoffs
8325
+ unreplayedCutoffs: cutoffs,
8326
+ liveDocuments: memoryMatchesTheTargets && cutoffs.size === 0 ? liveDocumentsOf(manager, targets) : null
8283
8327
  };
8284
8328
  } catch (error) {
8285
8329
  for (const plan of vectorPlans.values()) plan.release();
@@ -8542,6 +8586,25 @@ async function pagedTombstonesFor(directory, older, written, deleted) {
8542
8586
  return [...tombstones];
8543
8587
  }
8544
8588
 
8589
+ // src/persistence/durability/stored-bytes.ts
8590
+ var NOTHING_SAVED = { bytes: 0, vectorBytes: 0, partitions: [] };
8591
+ function savedBytesOfManifest(manifest) {
8592
+ let bytes = 0;
8593
+ const partitions = [];
8594
+ for (const partition of manifest.partitions) {
8595
+ let partitionBytes = 0;
8596
+ for (const segment of partition.segments) partitionBytes += segment.bytes;
8597
+ partitions.push({ partitionId: partition.partitionId, bytes: partitionBytes });
8598
+ bytes += partitionBytes;
8599
+ }
8600
+ let vectorBytes = 0;
8601
+ for (const field of manifest.vectors) {
8602
+ for (const file of field.files) vectorBytes += file.bytes;
8603
+ for (const part of field.graphParts) vectorBytes += part.bytes;
8604
+ }
8605
+ return { bytes: bytes + vectorBytes, vectorBytes, partitions };
8606
+ }
8607
+
8545
8608
  // src/persistence/durability/segment/build-segment.ts
8546
8609
  function writeRecordsOf(documents, written) {
8547
8610
  const seqNos = new Float64Array(documents.length);
@@ -10487,6 +10550,9 @@ async function mergePagedWindow(input) {
10487
10550
  for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
10488
10551
  }
10489
10552
  }
10553
+ for (const docId of input.replacedAfterWindow ?? []) {
10554
+ for (const source of sources) source.segment.tombstoneDocument(docId);
10555
+ }
10490
10556
  const tombstones = /* @__PURE__ */ new Set();
10491
10557
  if (!input.dropsEveryTombstone) {
10492
10558
  for (const source of sources) {
@@ -10580,49 +10646,34 @@ function retainedTombstones(frozen, contents) {
10580
10646
  }
10581
10647
  return [...retained];
10582
10648
  }
10583
- async function compactPartitionSegments(input) {
10584
- const { segments, compactionThreshold } = input;
10585
- if (segments.length <= compactionThreshold) {
10586
- return { segments, nextSegmentId: input.nextSegmentId };
10587
- }
10588
- const paged = input.paged === true;
10589
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
10590
- if (window === null) {
10591
- return { segments, nextSegmentId: input.nextSegmentId };
10592
- }
10593
- const windowRefs = segments.slice(window.start, window.start + window.windowLength);
10594
- const replacing = (written2) => ({
10595
- segments: [
10596
- ...segments.slice(0, window.start),
10597
- ...written2.segments,
10598
- ...segments.slice(window.start + window.windowLength)
10599
- ],
10600
- nextSegmentId: written2.nextSegmentId
10601
- });
10602
- if (paged) {
10649
+ async function mergeSegmentWindow(input, window) {
10650
+ if (input.paged === true) {
10603
10651
  const { directory } = input.store;
10604
- if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
10605
- return replacing(
10606
- await mergePagedWindow({
10607
- store: input.store,
10608
- directory,
10609
- indexName: input.indexName,
10610
- partitionId: input.partitionId,
10611
- schema: input.schema,
10612
- partialWordAnalysis: input.partialWordAnalysis,
10613
- textAnalysis: input.textAnalysis,
10614
- nextSegmentId: input.nextSegmentId,
10615
- refs: windowRefs,
10616
- dropsEveryTombstone: window.start === 0
10617
- })
10618
- );
10652
+ if (directory === void 0) return null;
10653
+ return mergePagedWindow({
10654
+ store: input.store,
10655
+ directory,
10656
+ indexName: input.indexName,
10657
+ partitionId: input.partitionId,
10658
+ schema: input.schema,
10659
+ partialWordAnalysis: input.partialWordAnalysis,
10660
+ textAnalysis: input.textAnalysis,
10661
+ nextSegmentId: input.nextSegmentId,
10662
+ refs: window.refs,
10663
+ dropsEveryTombstone: window.startsTheList,
10664
+ ...window.replacedAfterWindow === void 0 ? {} : { replacedAfterWindow: window.replacedAfterWindow }
10665
+ });
10619
10666
  }
10620
10667
  const contents = [];
10621
- for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
10668
+ for (const ref of window.refs) contents.push(await readSegmentFile(input.store, ref));
10622
10669
  const frozen = contents.map(frozenOf);
10623
10670
  applyTimeOrder(frozen, contents);
10671
+ for (const docId of window.replacedAfterWindow ?? []) {
10672
+ for (const segment of frozen) segment.tombstoneDocument(docId);
10673
+ }
10624
10674
  const writeRecords = writeRecordsOf2(frozen, contents);
10625
- const written = await writeSegmentsOf({
10675
+ const keepsTombstones = !(window.startsTheList && window.replacedAfterWindow !== void 0);
10676
+ return writeSegmentsOf({
10626
10677
  store: input.store,
10627
10678
  indexName: input.indexName,
10628
10679
  partitionId: input.partitionId,
@@ -10631,11 +10682,33 @@ async function compactPartitionSegments(input) {
10631
10682
  partialWordAnalysis: input.partialWordAnalysis,
10632
10683
  textAnalysis: input.textAnalysis,
10633
10684
  selections: frozen.map((segment) => ({ segment })),
10634
- tombstones: retainedTombstones(frozen, contents),
10685
+ tombstones: keepsTombstones ? retainedTombstones(frozen, contents) : [],
10635
10686
  mergesIntoOneFile: true,
10636
10687
  ...writeRecords === void 0 ? {} : { writeRecords }
10637
10688
  });
10638
- return replacing(written);
10689
+ }
10690
+ async function compactPartitionSegments(input) {
10691
+ const { segments, compactionThreshold } = input;
10692
+ if (segments.length <= compactionThreshold) {
10693
+ return { segments, nextSegmentId: input.nextSegmentId };
10694
+ }
10695
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, input.paged === true);
10696
+ if (window === null) {
10697
+ return { segments, nextSegmentId: input.nextSegmentId };
10698
+ }
10699
+ const written = await mergeSegmentWindow(input, {
10700
+ refs: segments.slice(window.start, window.start + window.windowLength),
10701
+ startsTheList: window.start === 0
10702
+ });
10703
+ if (written === null) return { segments, nextSegmentId: input.nextSegmentId };
10704
+ return {
10705
+ segments: [
10706
+ ...segments.slice(0, window.start),
10707
+ ...written.segments,
10708
+ ...segments.slice(window.start + window.windowLength)
10709
+ ],
10710
+ nextSegmentId: written.nextSegmentId
10711
+ };
10639
10712
  }
10640
10713
 
10641
10714
  // src/persistence/durability/segment/manifest-encode.ts
@@ -10684,6 +10757,161 @@ function encodeSegmentManifest(manifest) {
10684
10757
  return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
10685
10758
  }
10686
10759
 
10760
+ // src/persistence/paged/segment-scan.ts
10761
+ async function openPagedSegmentScan(directory, ref) {
10762
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
10763
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10764
+ const segment = createPagedFrozenSegment(snapshot, cache);
10765
+ const { paged } = segment;
10766
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10767
+ const ids = segment.arrays.idTable;
10768
+ const tombstones = new Set(snapshot.tombstones);
10769
+ return {
10770
+ count: snapshot.documentCount,
10771
+ schema: snapshot.schema,
10772
+ tombstones,
10773
+ idAt: (ordinal) => ids.idAt(ordinal),
10774
+ holds: (docId) => ids.ordinalOf(docId) >= 0,
10775
+ storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
10776
+ close: () => {
10777
+ paged.close();
10778
+ cache.clear();
10779
+ }
10780
+ };
10781
+ }
10782
+ async function checkPagedSegmentFully(directory, ref) {
10783
+ const listing = await pagedListingOf(directory, ref);
10784
+ await requirePagedPayloadChecksum(directory, listing);
10785
+ const snapshot = openPagedSnapshot(listing);
10786
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10787
+ const segment = createPagedFrozenSegment(snapshot, cache);
10788
+ const { paged } = segment;
10789
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10790
+ try {
10791
+ await checkPagedContents(segment, cache);
10792
+ return snapshot.schema;
10793
+ } finally {
10794
+ paged.close();
10795
+ cache.clear();
10796
+ }
10797
+ }
10798
+ async function bytesThatSegmentsHoldInMemory(directory, manifest) {
10799
+ let bytes = 0;
10800
+ for (const partition of manifest.partitions) {
10801
+ for (const ref of partition.segments) {
10802
+ const paged = await isPagedSegmentFile(directory, ref.key);
10803
+ bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
10804
+ }
10805
+ }
10806
+ return bytes;
10807
+ }
10808
+ function countsAsLatestHolder(scans, at, docId) {
10809
+ for (let later = at + 1; later < scans.length; later++) {
10810
+ if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
10811
+ }
10812
+ return true;
10813
+ }
10814
+
10815
+ // src/persistence/durability/segment/removed-documents.ts
10816
+ var decoder2 = new TextDecoder();
10817
+ async function readSegmentIds(store, ref) {
10818
+ const { directory } = store;
10819
+ if (directory !== void 0 && await isPagedSegmentFile(directory, ref.key)) {
10820
+ const scan = await openPagedSegmentScan(directory, ref);
10821
+ try {
10822
+ const ids2 = new Array(scan.count);
10823
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) ids2[ordinal] = scan.idAt(ordinal);
10824
+ return { ids: ids2, tombstones: [...scan.tombstones] };
10825
+ } finally {
10826
+ scan.close();
10827
+ }
10828
+ }
10829
+ const contents = await readSegmentFile(store, ref);
10830
+ const table = contents.arrays.idTable;
10831
+ const ids = new Array(contents.arrays.documentCount);
10832
+ for (let ordinal = 0; ordinal < ids.length; ordinal++) {
10833
+ ids[ordinal] = decoder2.decode(table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]));
10834
+ }
10835
+ return { ids, tombstones: contents.tombstones };
10836
+ }
10837
+ async function documentsReplacedLater(store, segments) {
10838
+ const later = /* @__PURE__ */ new Set();
10839
+ const replaced = new Array(segments.length);
10840
+ for (let at = segments.length - 1; at >= 0; at--) {
10841
+ const { ids, tombstones } = await readSegmentIds(store, segments[at]);
10842
+ const found = [];
10843
+ for (const docId of ids) if (later.has(docId)) found.push(docId);
10844
+ replaced[at] = found;
10845
+ if (at === 0) break;
10846
+ for (const docId of ids) later.add(docId);
10847
+ for (const docId of tombstones) later.add(docId);
10848
+ }
10849
+ return replaced;
10850
+ }
10851
+ function exceedsRemovedShare(removed, held) {
10852
+ return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
10853
+ }
10854
+ function freedBytesOf(ref, replacedCount) {
10855
+ return ref.docCount === 0 ? 0 : ref.bytes * replacedCount / ref.docCount;
10856
+ }
10857
+ function chooseRewrites(segments, replaced, budget) {
10858
+ let held = 0;
10859
+ let removed = 0;
10860
+ for (let at = 0; at < segments.length; at++) {
10861
+ held += segments[at].docCount;
10862
+ removed += replaced[at].length;
10863
+ }
10864
+ const candidates = [];
10865
+ for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
10866
+ candidates.sort(
10867
+ (left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
10868
+ );
10869
+ const chosen = [];
10870
+ for (const at of candidates) {
10871
+ if (!exceedsRemovedShare(removed, held)) break;
10872
+ const bytes = segments[at].bytes;
10873
+ if (chosen.length > 0 && budget.spentBytes + bytes > budget.limitBytes) continue;
10874
+ chosen.push(at);
10875
+ budget.spentBytes += bytes;
10876
+ held -= replaced[at].length;
10877
+ removed -= replaced[at].length;
10878
+ }
10879
+ return chosen.sort((left, right) => left - right);
10880
+ }
10881
+ function holdsSomething(ref) {
10882
+ return ref.docCount > 0 || ref.tombstoneCount > 0;
10883
+ }
10884
+ async function rewriteRemovedDocuments(input) {
10885
+ const { segments, liveDocumentCount } = input;
10886
+ const unchanged = { segments, nextSegmentId: input.nextSegmentId };
10887
+ if (liveDocumentCount === void 0) return unchanged;
10888
+ let held = 0;
10889
+ for (const ref of segments) held += ref.docCount;
10890
+ if (!exceedsRemovedShare(held - liveDocumentCount, held)) return unchanged;
10891
+ const replaced = await documentsReplacedLater(input.store, segments);
10892
+ const chosen = new Set(chooseRewrites(segments, replaced, input.budget));
10893
+ if (chosen.size === 0) return unchanged;
10894
+ const rewritten = [];
10895
+ let nextSegmentId = input.nextSegmentId;
10896
+ for (let at = 0; at < segments.length; at++) {
10897
+ if (!chosen.has(at)) {
10898
+ rewritten.push(segments[at]);
10899
+ continue;
10900
+ }
10901
+ const written = await mergeSegmentWindow(
10902
+ { ...input, nextSegmentId },
10903
+ { refs: [segments[at]], startsTheList: rewritten.length === 0, replacedAfterWindow: replaced[at] }
10904
+ );
10905
+ if (written === null) {
10906
+ rewritten.push(segments[at]);
10907
+ continue;
10908
+ }
10909
+ nextSegmentId = written.nextSegmentId;
10910
+ rewritten.push(...written.segments.filter(holdsSomething));
10911
+ }
10912
+ return { segments: rewritten, nextSegmentId };
10913
+ }
10914
+
10687
10915
  // src/persistence/durability/segment/write.ts
10688
10916
  function textAnalysisOf(config) {
10689
10917
  let resolved = null;
@@ -10806,8 +11034,21 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
10806
11034
  compactionThreshold: context.compactionThreshold,
10807
11035
  paged: context.input.metadata.textStorage === "disk"
10808
11036
  });
10809
- segments = compacted.segments;
10810
- nextSegmentId = compacted.nextSegmentId;
11037
+ const rewritten = await rewriteRemovedDocuments({
11038
+ store: context.input.store,
11039
+ indexName: context.indexName,
11040
+ partitionId,
11041
+ schema: context.input.metadata.schema,
11042
+ partialWordAnalysis: context.partialWordAnalysis,
11043
+ textAnalysis: context.textAnalysis,
11044
+ segments: compacted.segments,
11045
+ nextSegmentId: compacted.nextSegmentId,
11046
+ paged: context.input.metadata.textStorage === "disk",
11047
+ liveDocumentCount: context.input.liveDocuments?.find((entry) => entry.partitionId === partitionId)?.documentCount,
11048
+ budget: context.rewriteBudget
11049
+ });
11050
+ segments = rewritten.segments;
11051
+ nextSegmentId = rewritten.nextSegmentId;
10811
11052
  return { partitionId, nextSegmentId, segments };
10812
11053
  }
10813
11054
  function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
@@ -10832,7 +11073,11 @@ async function writeCheckpointSegments(input) {
10832
11073
  partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
10833
11074
  textAnalysis,
10834
11075
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
10835
- compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
11076
+ compactionThreshold: resolveCompactionThreshold(input.compactionThreshold),
11077
+ rewriteBudget: {
11078
+ spentBytes: 0,
11079
+ limitBytes: input.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT
11080
+ }
10836
11081
  };
10837
11082
  const checkpointByPartition = /* @__PURE__ */ new Map();
10838
11083
  for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
@@ -10869,6 +11114,7 @@ async function writeCheckpointSegments(input) {
10869
11114
  async function commitCheckpointManifest(store, metadata, written, vectors) {
10870
11115
  const indexName = metadata.indexName;
10871
11116
  const manifest = {
11117
+ version: SEGMENT_MANIFEST_VERSION,
10872
11118
  schema: metadata.schema,
10873
11119
  language: metadata.language,
10874
11120
  ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
@@ -10878,9 +11124,9 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
10878
11124
  };
10879
11125
  const packed = await encodeSegmentManifest(manifest);
10880
11126
  await store.write(manifestKey(indexName), packed.chunks);
10881
- return { garbage: await unreferencedKeys(store, indexName, manifest) };
11127
+ return { manifest, saved: savedBytesOfManifest(manifest) };
10882
11128
  }
10883
- async function unreferencedKeys(store, indexName, manifest) {
11129
+ async function checkpointGarbage(store, indexName, manifest) {
10884
11130
  const referenced = manifestReferencedKeys(manifest);
10885
11131
  const stored = await store.list(segmentsPrefix(indexName));
10886
11132
  return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
@@ -10902,7 +11148,8 @@ function workerCanWriteEverySegment(input) {
10902
11148
  }
10903
11149
  async function writeIndexCheckpointSegments(input) {
10904
11150
  const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
10905
- const { keepsWriteRecords } = input;
11151
+ const { keepsWriteRecords, liveDocuments, rewriteBytesPerCheckpoint } = input;
11152
+ const counted = liveDocuments === null ? {} : { liveDocuments, rewriteBytesPerCheckpoint };
10906
11153
  const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
10907
11154
  root: directory.root,
10908
11155
  metadata,
@@ -10910,7 +11157,8 @@ async function writeIndexCheckpointSegments(input) {
10910
11157
  compactionThreshold,
10911
11158
  prior,
10912
11159
  capturedPartitions: [...capturedPartitions],
10913
- keepsWriteRecords
11160
+ keepsWriteRecords,
11161
+ ...counted
10914
11162
  }) : null;
10915
11163
  if (offloaded !== null) {
10916
11164
  return offloaded;
@@ -10924,7 +11172,8 @@ async function writeIndexCheckpointSegments(input) {
10924
11172
  prior,
10925
11173
  wholePartitions,
10926
11174
  capturedPartitions,
10927
- keepsWriteRecords
11175
+ keepsWriteRecords,
11176
+ ...counted
10928
11177
  });
10929
11178
  }
10930
11179
 
@@ -11007,61 +11256,6 @@ async function startReadingOpeningSectionsAhead(directory, manifest) {
11007
11256
  }
11008
11257
  }
11009
11258
 
11010
- // src/persistence/paged/segment-scan.ts
11011
- async function openPagedSegmentScan(directory, ref) {
11012
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
11013
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
11014
- const segment = createPagedFrozenSegment(snapshot, cache);
11015
- const { paged } = segment;
11016
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
11017
- const ids = segment.arrays.idTable;
11018
- const tombstones = new Set(snapshot.tombstones);
11019
- return {
11020
- count: snapshot.documentCount,
11021
- schema: snapshot.schema,
11022
- tombstones,
11023
- idAt: (ordinal) => ids.idAt(ordinal),
11024
- holds: (docId) => ids.ordinalOf(docId) >= 0,
11025
- storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
11026
- close: () => {
11027
- paged.close();
11028
- cache.clear();
11029
- }
11030
- };
11031
- }
11032
- async function checkPagedSegmentFully(directory, ref) {
11033
- const listing = await pagedListingOf(directory, ref);
11034
- await requirePagedPayloadChecksum(directory, listing);
11035
- const snapshot = openPagedSnapshot(listing);
11036
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
11037
- const segment = createPagedFrozenSegment(snapshot, cache);
11038
- const { paged } = segment;
11039
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
11040
- try {
11041
- await checkPagedContents(segment, cache);
11042
- return snapshot.schema;
11043
- } finally {
11044
- paged.close();
11045
- cache.clear();
11046
- }
11047
- }
11048
- async function bytesThatSegmentsHoldInMemory(directory, manifest) {
11049
- let bytes = 0;
11050
- for (const partition of manifest.partitions) {
11051
- for (const ref of partition.segments) {
11052
- const paged = await isPagedSegmentFile(directory, ref.key);
11053
- bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
11054
- }
11055
- }
11056
- return bytes;
11057
- }
11058
- function countsAsLatestHolder(scans, at, docId) {
11059
- for (let later = at + 1; later < scans.length; later++) {
11060
- if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
11061
- }
11062
- return true;
11063
- }
11064
-
11065
11259
  // src/persistence/durability/segment/load-partition.ts
11066
11260
  function requireAgreedSchema(agreed, schema, key) {
11067
11261
  for (const [field, type] of Object.entries(schema)) {
@@ -11075,6 +11269,20 @@ function requireAgreedSchema(agreed, schema, key) {
11075
11269
  }
11076
11270
  }
11077
11271
  }
11272
+ function requireListedCounts(ref, documentCount, tombstoneCount) {
11273
+ if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
11274
+ throw new NarsilError(
11275
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
11276
+ `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
11277
+ {
11278
+ key: ref.key,
11279
+ documentCount,
11280
+ tombstoneCount,
11281
+ listedDocCount: ref.docCount,
11282
+ listedTombstoneCount: ref.tombstoneCount
11283
+ }
11284
+ );
11285
+ }
11078
11286
  function removeWhatTheSegmentReplaces(manager, segment, contents) {
11079
11287
  manager.beginBatchRemove();
11080
11288
  try {
@@ -11109,6 +11317,7 @@ function removeFromPartition(manager, partitionId, docIds) {
11109
11317
  async function attachPagedSegment(directory, ref, partitionId, manager, options) {
11110
11318
  if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
11111
11319
  const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
11320
+ requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
11112
11321
  requireAgreedSchema(options.schema, snapshot.schema, ref.key);
11113
11322
  const segment = createPagedFrozenSegment(snapshot);
11114
11323
  removeFromPartition(manager, partitionId, snapshot.tombstones);
@@ -11154,6 +11363,7 @@ async function attachPartitionSegments(store, partition, manager, options) {
11154
11363
  }
11155
11364
  }
11156
11365
  const contents = await readSegmentFile(store, ref);
11366
+ requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
11157
11367
  requireAgreedSchema(options.schema, contents.schema, ref.key);
11158
11368
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
11159
11369
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
@@ -11612,23 +11822,28 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11612
11822
  metadata,
11613
11823
  targets,
11614
11824
  compactionThreshold: input.compactionThreshold,
11825
+ rewriteBytesPerCheckpoint: input.rewriteBytesPerCheckpoint,
11615
11826
  canOffload: input.canOffload,
11616
11827
  prior: priorForLayout(priorManifest, capture3.partitionCount),
11617
11828
  wholePartitions: capture3.wholePartitions.contents,
11618
11829
  capturedPartitions: capture3.wholePartitions.captured,
11619
- keepsWriteRecords: writeRecords !== null
11830
+ keepsWriteRecords: writeRecords !== null,
11831
+ liveDocuments: capture3.liveDocuments
11620
11832
  })
11621
11833
  );
11622
11834
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
11623
11835
  if (aTargetFailed(input.indexState, targets)) return 0;
11624
11836
  const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
11837
+ input.recordSavedBytes(written.saved);
11625
11838
  input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
11626
11839
  recordWrittenVectors(vectorIndexes, liveVectors.written);
11627
11840
  await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
11628
11841
  const changes = changesSinceCapture();
11629
11842
  const droppedThisCheckpoint = changes !== null && capture3.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
11843
+ await writeCheckpointMetadata(input, documentCount, capture3.partitionCount, written.saved.bytes);
11630
11844
  if (!input.holdsCheckpointFiles()) {
11631
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
11845
+ const garbage = await checkpointGarbage(store, input.indexName, written.manifest);
11846
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, garbage);
11632
11847
  const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
11633
11848
  const keptForSync = await input.keepsCheckpointKey();
11634
11849
  await removeCheckpointGarbage(
@@ -11636,19 +11851,6 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11636
11851
  removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
11637
11852
  );
11638
11853
  }
11639
- const checkpointDocumentCount = documentCount;
11640
- await input.queueMetadataWrite(input.indexName, async () => {
11641
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
11642
- if (checkpointMetadata === void 0) {
11643
- return;
11644
- }
11645
- const bytes = await writeMetadataEnvelope(
11646
- { ...checkpointMetadata, partitionCount: capture3.partitionCount },
11647
- { checksum: true }
11648
- );
11649
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
11650
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture3.partitionCount);
11651
- });
11652
11854
  const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
11653
11855
  await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
11654
11856
  if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
@@ -11666,6 +11868,18 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11666
11868
  const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
11667
11869
  return anotherIsDue ? eligibleBeyond : capture3.recordsLeftInLog;
11668
11870
  }
11871
+ async function writeCheckpointMetadata(input, documentCount, partitionCount, savedBytes) {
11872
+ const { indexName, hooks } = input;
11873
+ await input.queueMetadataWrite(indexName, async () => {
11874
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
11875
+ if (checkpointMetadata === void 0) {
11876
+ return;
11877
+ }
11878
+ const bytes = await writeMetadataEnvelope({ ...checkpointMetadata, partitionCount }, { checksum: true });
11879
+ await input.directory.atomicWrite(`${indexName}/meta`, bytes);
11880
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, savedBytes);
11881
+ });
11882
+ }
11669
11883
  function recordsBeyond(indexState, targets, cutoffs) {
11670
11884
  let records = 0;
11671
11885
  for (const [partitionId, partition] of indexState.partitions) {
@@ -11830,6 +12044,258 @@ function createHeldCheckpoints(hooks) {
11830
12044
  };
11831
12045
  }
11832
12046
 
12047
+ // src/persistence/durability/index-stored-sizes.ts
12048
+ function refreshStoredSizes(sizes) {
12049
+ const known = sizes.saved !== null || !sizes.measuring && sizes.held;
12050
+ sizes.known = known;
12051
+ if (known) {
12052
+ sizes.savedBytes = sizes.saved?.bytes ?? 0;
12053
+ sizes.vectorSavedBytes = sizes.saved?.vectorBytes ?? 0;
12054
+ sizes.logBytes = sizes.tracksLog ? sizes.logged?.bytes ?? 0 : void 0;
12055
+ return;
12056
+ }
12057
+ sizes.savedBytes = sizes.measuring ? void 0 : sizes.recorded;
12058
+ sizes.vectorSavedBytes = void 0;
12059
+ sizes.logBytes = void 0;
12060
+ }
12061
+ function savedBytesByPartitionOf(saved) {
12062
+ const byPartition = [];
12063
+ if (saved === null) return byPartition;
12064
+ for (const { partitionId, bytes } of saved.partitions) byPartition[partitionId] = bytes;
12065
+ return byPartition;
12066
+ }
12067
+ function createTrackedSizes(tracksLog) {
12068
+ const sizes = {
12069
+ savedBytes: void 0,
12070
+ logBytes: void 0,
12071
+ vectorSavedBytes: void 0,
12072
+ saved: null,
12073
+ savedBytesByPartition: [],
12074
+ measuring: false,
12075
+ held: false,
12076
+ recorded: void 0,
12077
+ logged: null,
12078
+ known: false,
12079
+ tracksLog,
12080
+ addTo(stats) {
12081
+ if (!sizes.known) return;
12082
+ const saved = sizes.savedBytesByPartition;
12083
+ const logged = sizes.tracksLog ? sizes.logged?.bytesByPartition ?? [] : null;
12084
+ for (const partition of stats) {
12085
+ partition.savedBytes = saved[partition.partitionId] ?? 0;
12086
+ if (logged !== null) partition.logBytes = logged[partition.partitionId] ?? 0;
12087
+ }
12088
+ },
12089
+ recordSavedBytes(bytes) {
12090
+ sizes.recorded = bytes;
12091
+ refreshStoredSizes(sizes);
12092
+ }
12093
+ };
12094
+ return sizes;
12095
+ }
12096
+ function createStoredSizesRecord(tracksLog) {
12097
+ const sizesByIndex = /* @__PURE__ */ new Map();
12098
+ function of(indexName) {
12099
+ let sizes = sizesByIndex.get(indexName);
12100
+ if (sizes === void 0) {
12101
+ sizes = createTrackedSizes(tracksLog);
12102
+ sizesByIndex.set(indexName, sizes);
12103
+ }
12104
+ return sizes;
12105
+ }
12106
+ function forget(indexName) {
12107
+ const sizes = sizesByIndex.get(indexName);
12108
+ if (sizes === void 0) return;
12109
+ sizes.saved = null;
12110
+ sizes.savedBytesByPartition = [];
12111
+ sizes.measuring = false;
12112
+ sizes.held = false;
12113
+ refreshStoredSizes(sizes);
12114
+ }
12115
+ return {
12116
+ of,
12117
+ setSaved(indexName, saved) {
12118
+ const sizes = of(indexName);
12119
+ sizes.saved = saved;
12120
+ sizes.savedBytesByPartition = savedBytesByPartitionOf(saved);
12121
+ refreshStoredSizes(sizes);
12122
+ },
12123
+ setMeasuring(indexName, measuring) {
12124
+ const sizes = of(indexName);
12125
+ sizes.measuring = measuring;
12126
+ refreshStoredSizes(sizes);
12127
+ },
12128
+ hold(indexName) {
12129
+ const sizes = of(indexName);
12130
+ if (!sizes.held) {
12131
+ sizes.held = true;
12132
+ refreshStoredSizes(sizes);
12133
+ }
12134
+ return sizes;
12135
+ },
12136
+ forget,
12137
+ drop(indexName) {
12138
+ forget(indexName);
12139
+ sizesByIndex.delete(indexName);
12140
+ }
12141
+ };
12142
+ }
12143
+
12144
+ // src/persistence/durability/log-bytes.ts
12145
+ var LOG_SEGMENT_KEY = /^(.+)\/wal\/(\d+)\/\d{16}$/;
12146
+ var HISTORY_SEGMENT_KEY = /^(.+)\/history\/(\d+)\/\d{16}-\d{16}$/;
12147
+ function loggedFileOf(key) {
12148
+ const match = LOG_SEGMENT_KEY.exec(key) ?? HISTORY_SEGMENT_KEY.exec(key);
12149
+ if (match === null) return null;
12150
+ const partitionId = Number.parseInt(match[2], 10);
12151
+ return Number.isSafeInteger(partitionId) ? { indexName: match[1], partitionId } : null;
12152
+ }
12153
+ function emptyLog() {
12154
+ return { files: /* @__PURE__ */ new Map(), bytes: 0, bytesByPartition: [], sizes: null };
12155
+ }
12156
+ function addBytes(log, partitionId, delta) {
12157
+ log.bytes += delta;
12158
+ log.bytesByPartition[partitionId] = (log.bytesByPartition[partitionId] ?? 0) + delta;
12159
+ if (log.sizes !== null) refreshStoredSizes(log.sizes);
12160
+ }
12161
+ function makeCurrent(previous, log, sizes) {
12162
+ if (previous !== void 0 && previous !== log) previous.sizes = null;
12163
+ log.sizes = sizes;
12164
+ sizes.logged = log;
12165
+ refreshStoredSizes(sizes);
12166
+ }
12167
+ function setFileBytes(log, key, partitionId, bytes) {
12168
+ const previous = log.files.get(key);
12169
+ if (previous !== void 0) addBytes(log, previous.partitionId, -previous.bytes);
12170
+ log.files.set(key, { partitionId, bytes });
12171
+ addBytes(log, partitionId, bytes);
12172
+ }
12173
+ function removeFile(log, key) {
12174
+ const previous = log.files.get(key);
12175
+ if (previous === void 0) return void 0;
12176
+ log.files.delete(key);
12177
+ addBytes(log, previous.partitionId, -previous.bytes);
12178
+ return previous;
12179
+ }
12180
+ function createLogBytesLedger(base, sizes) {
12181
+ const logs = /* @__PURE__ */ new Map();
12182
+ function logOf(indexName) {
12183
+ let log = logs.get(indexName);
12184
+ if (log === void 0) {
12185
+ log = emptyLog();
12186
+ logs.set(indexName, log);
12187
+ makeCurrent(void 0, log, sizes.of(indexName));
12188
+ }
12189
+ return log;
12190
+ }
12191
+ function track(key, bytes) {
12192
+ const file = loggedFileOf(key);
12193
+ if (file !== null) setFileBytes(logOf(file.indexName), key, file.partitionId, bytes);
12194
+ }
12195
+ function untrack(key) {
12196
+ const file = loggedFileOf(key);
12197
+ if (file === null) return void 0;
12198
+ const log = logs.get(file.indexName);
12199
+ return log === void 0 ? void 0 : removeFile(log, key);
12200
+ }
12201
+ async function currentLength(key) {
12202
+ return await base.byteLength?.(key) ?? (await base.read(key))?.byteLength ?? 0;
12203
+ }
12204
+ function trackedHandle(key, file, handle) {
12205
+ async function remeasure() {
12206
+ const log = logs.get(file.indexName);
12207
+ if (log?.files.has(key) === true) setFileBytes(log, key, file.partitionId, await handle.size());
12208
+ }
12209
+ return {
12210
+ async append(bytes) {
12211
+ try {
12212
+ await handle.append(bytes);
12213
+ } catch (error) {
12214
+ await remeasure().catch(() => void 0);
12215
+ throw error;
12216
+ }
12217
+ const log = logs.get(file.indexName);
12218
+ const entry = log?.files.get(key);
12219
+ if (log === void 0 || entry === void 0) return;
12220
+ entry.bytes += bytes.byteLength;
12221
+ addBytes(log, file.partitionId, bytes.byteLength);
12222
+ },
12223
+ sync: () => handle.sync(),
12224
+ async truncate(byteLength2) {
12225
+ try {
12226
+ await handle.truncate(byteLength2);
12227
+ } finally {
12228
+ await remeasure().catch(() => void 0);
12229
+ }
12230
+ },
12231
+ size: () => handle.size(),
12232
+ close: () => handle.close()
12233
+ };
12234
+ }
12235
+ const directory = {
12236
+ root: base.root,
12237
+ async appendHandle(key) {
12238
+ const handle = await base.appendHandle(key);
12239
+ const file = loggedFileOf(key);
12240
+ if (file === null) return handle;
12241
+ if (logs.get(file.indexName)?.files.has(key) !== true) track(key, await handle.size());
12242
+ return trackedHandle(key, file, handle);
12243
+ },
12244
+ markerHandle: (key) => base.markerHandle(key),
12245
+ async atomicWrite(key, data) {
12246
+ await base.atomicWrite(key, data);
12247
+ if (loggedFileOf(key) === null) return;
12248
+ let bytes = 0;
12249
+ for (const chunk of Array.isArray(data) ? data : [data]) bytes += chunk.byteLength;
12250
+ track(key, bytes);
12251
+ },
12252
+ syncDirectoryOf: (key) => base.syncDirectoryOf(key),
12253
+ read: (key) => base.read(key),
12254
+ async remove(key) {
12255
+ await base.remove(key);
12256
+ untrack(key);
12257
+ },
12258
+ async move(from, to) {
12259
+ await base.move(from, to);
12260
+ const moved = untrack(from);
12261
+ if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
12262
+ },
12263
+ list: (prefix) => base.list(prefix),
12264
+ pathOf: (key) => base.pathOf(key)
12265
+ };
12266
+ const { readRange, byteLength, identity, lock } = base;
12267
+ if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
12268
+ if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
12269
+ if (identity !== void 0) directory.identity = () => identity.call(base);
12270
+ if (lock !== void 0) directory.lock = () => lock.call(base);
12271
+ return {
12272
+ directory,
12273
+ async seed(indexName) {
12274
+ const log = emptyLog();
12275
+ for (const prefix of [`${indexName}/wal/`, `${indexName}/history/`]) {
12276
+ for (const key of await base.list(prefix)) {
12277
+ const file = loggedFileOf(key);
12278
+ if (file === null || file.indexName !== indexName) continue;
12279
+ setFileBytes(log, key, file.partitionId, await currentLength(key));
12280
+ }
12281
+ }
12282
+ const previous = logs.get(indexName);
12283
+ logs.set(indexName, log);
12284
+ makeCurrent(previous, log, sizes.of(indexName));
12285
+ },
12286
+ forget(indexName) {
12287
+ const log = logs.get(indexName);
12288
+ if (log === void 0) return;
12289
+ logs.delete(indexName);
12290
+ if (log.sizes !== null) {
12291
+ log.sizes.logged = null;
12292
+ refreshStoredSizes(log.sizes);
12293
+ log.sizes = null;
12294
+ }
12295
+ }
12296
+ };
12297
+ }
12298
+
11833
12299
  // src/persistence/durability/manager-state.ts
11834
12300
  function closePartitionWriter(partition) {
11835
12301
  return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
@@ -12244,6 +12710,7 @@ function createIndexUnloading(deps) {
12244
12710
  }
12245
12711
  await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
12246
12712
  deps.forgetSyncState(indexName);
12713
+ deps.dropStoredBytes(indexName);
12247
12714
  },
12248
12715
  async unloadIndex(indexName) {
12249
12716
  const indexState = indexes.get(indexName);
@@ -12254,6 +12721,7 @@ function createIndexUnloading(deps) {
12254
12721
  indexes.delete(indexName);
12255
12722
  metadataWrites.forget(indexName);
12256
12723
  deps.forgetSyncState(indexName);
12724
+ deps.forgetStoredBytes(indexName);
12257
12725
  }
12258
12726
  };
12259
12727
  }
@@ -12721,7 +13189,7 @@ function createSyncStorage(deps) {
12721
13189
 
12722
13190
  // src/persistence/durability/manager-storage.ts
12723
13191
  function createIndexStorageAccess(deps) {
12724
- const { directory, indexes } = deps;
13192
+ const { directory, logBytes, sizes } = deps;
12725
13193
  const archiveAccess = createArchiveCheckpointAccess({
12726
13194
  directory,
12727
13195
  held: deps.heldCheckpointFiles,
@@ -12733,7 +13201,7 @@ function createIndexStorageAccess(deps) {
12733
13201
  directory,
12734
13202
  openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
12735
13203
  inCheckpointTurn: (indexName, work) => {
12736
- const indexState = indexes.get(indexName);
13204
+ const indexState = deps.indexes.get(indexName);
12737
13205
  if (indexState === void 0) return work();
12738
13206
  return checkpointInTurn(indexState, {
12739
13207
  joinsInFlight: false,
@@ -12746,12 +13214,54 @@ function createIndexStorageAccess(deps) {
12746
13214
  const unloading = createIndexUnloading({
12747
13215
  directory,
12748
13216
  hooks: deps.hooks,
12749
- indexes,
13217
+ indexes: deps.indexes,
12750
13218
  metadataWrites: deps.metadataWrites,
12751
13219
  closeWriterReportingFailure: deps.closeWriterReportingFailure,
12752
- forgetSyncState: syncStorage.forget
13220
+ forgetSyncState: syncStorage.forget,
13221
+ forgetStoredBytes: (indexName) => {
13222
+ sizes.forget(indexName);
13223
+ logBytes.forget(indexName);
13224
+ },
13225
+ dropStoredBytes: (indexName) => {
13226
+ sizes.drop(indexName);
13227
+ logBytes.forget(indexName);
13228
+ }
12753
13229
  });
12754
- return { archiveAccess, syncStorage, unloading };
13230
+ async function measureStoredBytes(indexName, saved) {
13231
+ sizes.setSaved(indexName, saved);
13232
+ await logBytes.seed(indexName);
13233
+ }
13234
+ const recoveryDeps = {
13235
+ directory,
13236
+ hooks: deps.hooks,
13237
+ compactionThreshold: deps.compactionThreshold,
13238
+ registerPartition: deps.registerPartition,
13239
+ holdsCheckpointFiles: (indexName) => deps.heldCheckpointFiles.has(indexName),
13240
+ queueMetadataWrite: deps.metadataWrites.queue,
13241
+ syncStorage,
13242
+ measureStoredBytes
13243
+ };
13244
+ const storedBytesAccess = {
13245
+ storedSizesOf: sizes.of,
13246
+ addStoredSizes(indexName, stats) {
13247
+ sizes.of(indexName).addTo(stats);
13248
+ },
13249
+ vectorSavedBytes(indexName) {
13250
+ return sizes.of(indexName).vectorSavedBytes;
13251
+ },
13252
+ async measureRestoredStorage(indexName, restored) {
13253
+ if (restored !== void 0) {
13254
+ await measureStoredBytes(indexName, restored);
13255
+ return;
13256
+ }
13257
+ const decoded = await readSegmentManifest(directoryCheckpointStore(directory), indexName);
13258
+ await measureStoredBytes(
13259
+ indexName,
13260
+ decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED
13261
+ );
13262
+ }
13263
+ };
13264
+ return { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps };
12755
13265
  }
12756
13266
 
12757
13267
  // src/persistence/durability/metadata-write-queue.ts
@@ -13277,7 +13787,7 @@ function createWalWriter(directory, config) {
13277
13787
  }
13278
13788
 
13279
13789
  // src/persistence/durability/partition-registry.ts
13280
- function createPartitionRegistry(directory, segmentMaxBytes) {
13790
+ function createPartitionRegistry(directory, segmentMaxBytes, onIndexHeld) {
13281
13791
  const indexes = /* @__PURE__ */ new Map();
13282
13792
  function indexState(indexName) {
13283
13793
  let state = indexes.get(indexName);
@@ -13291,6 +13801,7 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
13291
13801
  stalledWrites: []
13292
13802
  };
13293
13803
  indexes.set(indexName, state);
13804
+ onIndexHeld(indexName);
13294
13805
  }
13295
13806
  return state;
13296
13807
  }
@@ -13826,7 +14337,7 @@ async function readEarlierSegmentDocuments(store, key) {
13826
14337
  }
13827
14338
 
13828
14339
  // src/persistence/durability/checkpoint-count.ts
13829
- var decoder2 = new TextDecoder();
14340
+ var decoder3 = new TextDecoder();
13830
14341
  async function countPagedPartition(store, partition) {
13831
14342
  const { directory } = store;
13832
14343
  if (directory === void 0 || partition.segments.length === 0) return null;
@@ -13859,7 +14370,7 @@ async function countCurrentLayout(store, manifest) {
13859
14370
  for (const docId of contents.tombstones) live.delete(docId);
13860
14371
  const ids = contents.arrays.idTable;
13861
14372
  for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
13862
- live.add(decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
14373
+ live.add(decoder3.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
13863
14374
  }
13864
14375
  }
13865
14376
  total += live.size;
@@ -14068,19 +14579,18 @@ async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
14068
14579
  const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
14069
14580
  recordWrittenVectors(deps.vectorIndexes, vectors.written);
14070
14581
  await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
14071
- await removeCheckpointGarbage(
14072
- deps.store,
14073
- await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, committed.garbage)
14074
- );
14582
+ const garbage = await checkpointGarbage(deps.store, deps.indexName, committed.manifest);
14583
+ await removeCheckpointGarbage(deps.store, await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, garbage));
14584
+ return committed.saved;
14075
14585
  }
14076
14586
  async function upgradeEarlierLayout(deps, earlier) {
14077
14587
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
14078
14588
  try {
14079
14589
  const loaded = await loadEarlier(deps, earlier);
14080
14590
  const targets = await deps.replay(loaded.checkpoint);
14081
- await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
14591
+ const saved = await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
14082
14592
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
14083
- return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
14593
+ return { checkpoint: targets, documentCount: deps.manager.countDocuments(), saved };
14084
14594
  } catch (error) {
14085
14595
  const failure = error instanceof Error ? error : new Error(String(error));
14086
14596
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
@@ -14100,11 +14610,11 @@ async function recoverCheckpoint(deps) {
14100
14610
  const earlier = await earlierLayoutOf(deps, decoded);
14101
14611
  if (earlier !== null) {
14102
14612
  const upgraded = await upgradeEarlierLayout(deps, earlier);
14103
- return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
14613
+ return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount, saved: upgraded.saved };
14104
14614
  }
14105
14615
  if (decoded === null || decoded.kind !== "current") {
14106
14616
  if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
14107
- return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
14617
+ return { checkpoint: await deps.replay([]), upgradedDocumentCount: null, saved: NOTHING_SAVED };
14108
14618
  }
14109
14619
  await deps.store.remove(snapshotBundleKey(deps.indexName));
14110
14620
  if (!deps.keepsUnlistedFiles) {
@@ -14117,7 +14627,11 @@ async function recoverCheckpoint(deps) {
14117
14627
  vectorIndexes: deps.vectorIndexes,
14118
14628
  writeRecords: deps.writeRecords ?? null
14119
14629
  });
14120
- return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
14630
+ return {
14631
+ checkpoint: await deps.replay(checkpoint),
14632
+ upgradedDocumentCount: null,
14633
+ saved: savedBytesOfManifest(decoded.manifest)
14634
+ };
14121
14635
  }
14122
14636
 
14123
14637
  // src/persistence/durability/layout-transition/recovery.ts
@@ -14219,9 +14733,9 @@ async function settleStoredAnalysis(store, metadata, metadataOnly) {
14219
14733
  }
14220
14734
 
14221
14735
  // src/persistence/durability/recover-index.ts
14222
- async function writeCountedMetadata(deps, indexName, documentCount) {
14736
+ async function writeCountedMetadata(deps, indexName, documentCount, savedBytes) {
14223
14737
  await deps.queueMetadataWrite(indexName, async () => {
14224
- const metadata = deps.hooks.buildMetadata(indexName, documentCount);
14738
+ const metadata = deps.hooks.buildMetadata(indexName, documentCount, savedBytes);
14225
14739
  if (metadata === void 0) return;
14226
14740
  const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
14227
14741
  await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
@@ -14316,8 +14830,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
14316
14830
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
14317
14831
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
14318
14832
  }
14833
+ await deps.measureStoredBytes(indexName, recovered.saved);
14319
14834
  if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
14320
- await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
14835
+ await writeCountedMetadata(
14836
+ deps,
14837
+ indexName,
14838
+ recovered.upgradedDocumentCount ?? manager.countDocuments(),
14839
+ recovered.saved.bytes
14840
+ );
14321
14841
  }
14322
14842
  return layout.transition === null ? null : removeUnread;
14323
14843
  }
@@ -14333,15 +14853,22 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
14333
14853
 
14334
14854
  // src/persistence/durability/manager.ts
14335
14855
  function createDurabilityManager(config, hooks, directoryOverride) {
14336
- const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
14856
+ const sizes = createStoredSizesRecord(true);
14857
+ const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
14858
+ const { directory } = logBytes;
14337
14859
  const canOffloadCheckpoint = directoryOverride === void 0;
14338
14860
  const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
14339
14861
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
14340
14862
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
14341
14863
  const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
14864
+ const rewriteBytesPerCheckpoint = config.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT;
14342
14865
  const mode = config.mode ?? "sync";
14343
14866
  const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
14344
- const registry = createPartitionRegistry(directory, segmentMaxBytes);
14867
+ const registry = createPartitionRegistry(
14868
+ directory,
14869
+ segmentMaxBytes,
14870
+ (indexName) => hooks.linkStoredSizes(indexName, sizes.hold(indexName))
14871
+ );
14345
14872
  const indexes = registry.indexes;
14346
14873
  const getOrCreateIndexState = registry.indexState;
14347
14874
  const getOrCreatePartition = registry.partition;
@@ -14432,12 +14959,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14432
14959
  indexName,
14433
14960
  indexState,
14434
14961
  compactionThreshold,
14962
+ rewriteBytesPerCheckpoint,
14435
14963
  canOffload: canOffloadCheckpoint,
14436
14964
  fromMemory,
14437
14965
  mutationThreshold: checkpointMutationThreshold,
14438
14966
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
14439
14967
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
14440
14968
  keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
14969
+ recordSavedBytes: (saved) => sizes.setSaved(indexName, saved),
14441
14970
  queueMetadataWrite,
14442
14971
  markFatal,
14443
14972
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -14492,28 +15021,22 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14492
15021
  startAsyncFlushTimer();
14493
15022
  return outcomes;
14494
15023
  }
14495
- const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
15024
+ const { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps } = createIndexStorageAccess({
14496
15025
  directory,
15026
+ logBytes,
15027
+ sizes,
14497
15028
  hooks,
14498
15029
  indexes,
14499
15030
  metadataWrites,
14500
15031
  heldCheckpointFiles,
14501
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
14502
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
14503
- markFatal,
14504
- closeWriterReportingFailure
14505
- });
14506
- const recoveryDeps = {
14507
- directory,
14508
- hooks,
14509
15032
  compactionThreshold,
15033
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
14510
15034
  registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
14511
15035
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
14512
15036
  },
14513
- holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
14514
- queueMetadataWrite,
14515
- syncStorage
14516
- };
15037
+ markFatal,
15038
+ closeWriterReportingFailure
15039
+ });
14517
15040
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
14518
15041
  const recovered = recoverWithLayoutCheckpoint(
14519
15042
  recoveryDeps,
@@ -14610,6 +15133,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14610
15133
  syncStorage: () => syncStorage,
14611
15134
  ...queryHistoryAccess({ kind: "directory", directory }),
14612
15135
  ...archiveAccess,
15136
+ ...storedBytesAccess,
14613
15137
  async shutdown() {
14614
15138
  shuttingDown = true;
14615
15139
  stopTimers();
@@ -14628,6 +15152,31 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14628
15152
  };
14629
15153
  }
14630
15154
 
15155
+ // src/persistence/durability/snapshot-only/index-state.ts
15156
+ function createSnapshotIndexState() {
15157
+ return {
15158
+ mutationsSinceCheckpoint: 0,
15159
+ checkpointInFlight: null,
15160
+ applyChain: Promise.resolve(),
15161
+ unloading: false,
15162
+ appliedSeqNoByPartition: /* @__PURE__ */ new Map()
15163
+ };
15164
+ }
15165
+ function metadataKey(indexName) {
15166
+ return `${indexName}/meta`;
15167
+ }
15168
+ function noteSnapshotCommitted(state, positions, partitionCount) {
15169
+ let recordsLeft = 0;
15170
+ for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
15171
+ if (partitionId >= partitionCount) {
15172
+ state.appliedSeqNoByPartition.delete(partitionId);
15173
+ continue;
15174
+ }
15175
+ recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
15176
+ }
15177
+ state.mutationsSinceCheckpoint = recordsLeft;
15178
+ }
15179
+
14631
15180
  // src/persistence/durability/snapshot-only/checkpoint.ts
14632
15181
  function capture(input, prior, cutoffs) {
14633
15182
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
@@ -14686,10 +15235,21 @@ async function writeCaptured(input, store, prior, captured) {
14686
15235
  });
14687
15236
  const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
14688
15237
  recordWrittenVectors(input.vectorIndexes, vectors.written);
14689
- await removeCheckpointGarbage(
14690
- store,
14691
- await garbageThatNoFieldStillReads(store, input.vectorIndexes, committed.garbage)
14692
- );
15238
+ await input.onCommitted({
15239
+ saved: committed.saved,
15240
+ documentCount: captured.documentCount,
15241
+ partitionCount: captured.partitionCount
15242
+ });
15243
+ const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
15244
+ await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
15245
+ }
15246
+ async function saveCheckpointMetadata(adapter, hooks, indexName, committed) {
15247
+ const { saved, documentCount, partitionCount } = committed;
15248
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, saved.bytes);
15249
+ if (checkpointMetadata === void 0) return;
15250
+ checkpointMetadata.partitionCount = partitionCount;
15251
+ await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(checkpointMetadata, { checksum: true }));
15252
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, saved.bytes);
14693
15253
  }
14694
15254
  async function writeSnapshotOnlyCheckpoint(input) {
14695
15255
  const store = adapterCheckpointStore(input.adapter);
@@ -14700,41 +15260,12 @@ async function writeSnapshotOnlyCheckpoint(input) {
14700
15260
  });
14701
15261
  try {
14702
15262
  await writeCaptured(input, store, prior, captured);
14703
- return {
14704
- documentCount: captured.documentCount,
14705
- positions: captured.positions,
14706
- partitionCount: captured.partitionCount
14707
- };
15263
+ return { positions: captured.positions, partitionCount: captured.partitionCount };
14708
15264
  } finally {
14709
15265
  for (const plan of captured.plans.values()) plan.release();
14710
15266
  }
14711
15267
  }
14712
15268
 
14713
- // src/persistence/durability/snapshot-only/index-state.ts
14714
- function createSnapshotIndexState() {
14715
- return {
14716
- mutationsSinceCheckpoint: 0,
14717
- checkpointInFlight: null,
14718
- applyChain: Promise.resolve(),
14719
- unloading: false,
14720
- appliedSeqNoByPartition: /* @__PURE__ */ new Map()
14721
- };
14722
- }
14723
- function metadataKey(indexName) {
14724
- return `${indexName}/meta`;
14725
- }
14726
- function noteSnapshotCommitted(state, positions, partitionCount) {
14727
- let recordsLeft = 0;
14728
- for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
14729
- if (partitionId >= partitionCount) {
14730
- state.appliedSeqNoByPartition.delete(partitionId);
14731
- continue;
14732
- }
14733
- recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
14734
- }
14735
- state.mutationsSinceCheckpoint = recordsLeft;
14736
- }
14737
-
14738
15269
  // src/persistence/durability/snapshot-only/restores.ts
14739
15270
  function toError6(error) {
14740
15271
  return error instanceof Error ? error : new Error(String(error));
@@ -14772,15 +15303,67 @@ function createPendingRestores(onFatalError) {
14772
15303
  };
14773
15304
  }
14774
15305
 
15306
+ // src/persistence/durability/snapshot-only/saved-size.ts
15307
+ function createSnapshotSavedSize(adapter, settleRestore) {
15308
+ const sizes = createStoredSizesRecord(false);
15309
+ const generations = /* @__PURE__ */ new Map();
15310
+ const store = adapterCheckpointStore(adapter);
15311
+ function replace(indexName, saved) {
15312
+ const generation = (generations.get(indexName) ?? 0) + 1;
15313
+ generations.set(indexName, generation);
15314
+ sizes.setSaved(indexName, saved);
15315
+ sizes.setMeasuring(indexName, saved === null);
15316
+ return generation;
15317
+ }
15318
+ async function measureAfterRestore(indexName, generation) {
15319
+ await settleRestore(indexName);
15320
+ const decoded = await readSegmentManifest(store, indexName);
15321
+ if (generations.get(indexName) !== generation) return;
15322
+ replace(indexName, decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED);
15323
+ }
15324
+ return {
15325
+ record(indexName, saved) {
15326
+ replace(indexName, saved);
15327
+ },
15328
+ hold: sizes.hold,
15329
+ forget(indexName) {
15330
+ generations.delete(indexName);
15331
+ sizes.forget(indexName);
15332
+ },
15333
+ drop(indexName) {
15334
+ generations.delete(indexName);
15335
+ sizes.drop(indexName);
15336
+ },
15337
+ access: {
15338
+ storedSizesOf: sizes.of,
15339
+ addStoredSizes(indexName, stats) {
15340
+ sizes.of(indexName).addTo(stats);
15341
+ },
15342
+ vectorSavedBytes(indexName) {
15343
+ return sizes.of(indexName).vectorSavedBytes;
15344
+ },
15345
+ async measureRestoredStorage(indexName, restored) {
15346
+ if (restored !== void 0) {
15347
+ replace(indexName, restored);
15348
+ return;
15349
+ }
15350
+ const generation = replace(indexName, null);
15351
+ void measureAfterRestore(indexName, generation).catch(() => void 0);
15352
+ }
15353
+ }
15354
+ };
15355
+ }
15356
+
14775
15357
  // src/persistence/durability/snapshot-only/index.ts
14776
15358
  function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14777
15359
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
14778
15360
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
14779
15361
  const indexes = /* @__PURE__ */ new Map();
14780
- const metadataWrites = /* @__PURE__ */ new Map();
15362
+ const metadataWrites = createMetadataWriteQueue();
14781
15363
  const loads = createIndexLoads(hooks.getVectorIndexes);
14782
15364
  const restoreStorage = adapterRestoreStorage(adapter);
14783
15365
  const restores = createPendingRestores(hooks.onFatalError);
15366
+ const savedSize = createSnapshotSavedSize(adapter, settleRestore);
14784
15367
  let checkpointTimer = null;
14785
15368
  let shuttingDown = false;
14786
15369
  function settleRestore(indexName) {
@@ -14791,6 +15374,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14791
15374
  if (state === void 0) {
14792
15375
  state = createSnapshotIndexState();
14793
15376
  indexes.set(indexName, state);
15377
+ hooks.linkStoredSizes(indexName, savedSize.hold(indexName));
14794
15378
  }
14795
15379
  return state;
14796
15380
  }
@@ -14819,23 +15403,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14819
15403
  const manager = hooks.getManager(indexName);
14820
15404
  const metadata = hooks.buildTermsMetadata(indexName);
14821
15405
  if (manager === void 0 || metadata === void 0) return;
14822
- const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
15406
+ const { positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
14823
15407
  adapter,
14824
15408
  indexName,
14825
15409
  metadata,
14826
15410
  manager,
14827
15411
  vectorIndexes: hooks.getVectorIndexes(indexName),
14828
15412
  appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
14829
- rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
14830
- });
14831
- hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
14832
- await queueMetadataWrite(indexName, async () => {
14833
- const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
14834
- if (checkpointMetadata === void 0) return;
14835
- checkpointMetadata.partitionCount = partitionCount;
14836
- const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
14837
- await adapter.save(metadataKey(indexName), metadataBytes);
14838
- hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
15413
+ rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
15414
+ onCommitted: async (committed) => {
15415
+ savedSize.record(indexName, committed.saved);
15416
+ hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
15417
+ await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
15418
+ }
14839
15419
  });
14840
15420
  noteSnapshotCommitted(indexState, positions, partitionCount);
14841
15421
  if (publisher !== void 0) {
@@ -14861,16 +15441,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14861
15441
  );
14862
15442
  }
14863
15443
  function queueMetadataWrite(indexName, write) {
14864
- const previous = metadataWrites.get(indexName) ?? Promise.resolve();
14865
- const queued = previous.then(() => settleRestore(indexName)).then(write);
14866
- metadataWrites.set(
14867
- indexName,
14868
- queued.catch(() => void 0)
14869
- );
14870
- return queued;
15444
+ return metadataWrites.queue(indexName, () => settleRestore(indexName).then(write));
14871
15445
  }
14872
- async function saveMetadata(indexName, documentCount) {
14873
- const metadata = hooks.buildMetadata(indexName, documentCount);
15446
+ async function saveMetadata(indexName, documentCount, savedBytes) {
15447
+ const metadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
14874
15448
  if (metadata === void 0) return;
14875
15449
  await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
14876
15450
  }
@@ -14880,9 +15454,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14880
15454
  async function loadIndex(indexName, metadataOnly) {
14881
15455
  await settleRestore(indexName);
14882
15456
  const metaBytes = await adapter.load(metadataKey(indexName));
14883
- if (metaBytes === null) {
14884
- return;
14885
- }
15457
+ if (metaBytes === null) return;
14886
15458
  const { metadata } = await readMetadataEnvelope(metaBytes);
14887
15459
  const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
14888
15460
  let derivedCount;
@@ -14921,6 +15493,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14921
15493
  onLayoutUpgrade: hooks.onLayoutUpgrade
14922
15494
  });
14923
15495
  const indexState = getOrCreateIndexState(indexName);
15496
+ savedSize.record(indexName, recovered.saved);
14924
15497
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
14925
15498
  const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
14926
15499
  if (persistedSeqNo > 0) {
@@ -14929,7 +15502,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14929
15502
  }
14930
15503
  if (recovered.upgradedDocumentCount !== null || promoted) {
14931
15504
  const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
14932
- await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
15505
+ await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount, recovered.saved.bytes)).catch(
15506
+ () => void 0
15507
+ );
14933
15508
  }
14934
15509
  }
14935
15510
  async function recordMutation(record) {
@@ -14961,14 +15536,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14961
15536
  try {
14962
15537
  await indexState.applyChain;
14963
15538
  if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
14964
- await metadataWrites.get(indexName);
15539
+ await metadataWrites.pending(indexName);
14965
15540
  } catch (error) {
14966
15541
  indexState.unloading = false;
14967
15542
  throw error;
14968
15543
  }
14969
15544
  }
14970
15545
  indexes.delete(indexName);
14971
- metadataWrites.delete(indexName);
15546
+ metadataWrites.forget(indexName);
15547
+ savedSize.forget(indexName);
14972
15548
  }
14973
15549
  return {
14974
15550
  isActive() {
@@ -15050,6 +15626,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
15050
15626
  }
15051
15627
  },
15052
15628
  ...queryHistoryAccess({ kind: "adapter", adapter }),
15629
+ ...savedSize.access,
15053
15630
  async removeIndex(indexName) {
15054
15631
  await settleRestore(indexName).catch(() => void 0);
15055
15632
  restores.forget(indexName);
@@ -15059,6 +15636,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
15059
15636
  await adapter.delete(key);
15060
15637
  }
15061
15638
  await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
15639
+ savedSize.drop(indexName);
15062
15640
  },
15063
15641
  async unloadIndex(indexName) {
15064
15642
  await settleRestore(indexName);
@@ -15103,7 +15681,7 @@ function vectorFieldsOf(schema, promotion) {
15103
15681
  }
15104
15682
  return Object.keys(fields).length === 0 ? null : fields;
15105
15683
  }
15106
- function metadataFromConfig(indexName, config, partitionCount, documentCount) {
15684
+ function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
15107
15685
  const metadata = {
15108
15686
  indexName,
15109
15687
  schema: config.schema,
@@ -15118,6 +15696,8 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
15118
15696
  engineVersion: VERSION
15119
15697
  };
15120
15698
  metadata.documentCount = documentCount ?? config.documentCount;
15699
+ const recordedSavedBytes = savedBytes ?? config.savedBytes;
15700
+ if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
15121
15701
  const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
15122
15702
  if (vectorFields !== null) {
15123
15703
  metadata.vectorFields = vectorFields;
@@ -15197,6 +15777,7 @@ function metadataConfigOf(entry) {
15197
15777
  ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
15198
15778
  ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
15199
15779
  documentCount: entry.documentCount,
15780
+ savedBytes: entry.savedBytes,
15200
15781
  partitionCount: entry.partitionCount,
15201
15782
  schema: flattenSchema(entry.config.schema),
15202
15783
  language: entry.language.name,
@@ -15275,7 +15856,7 @@ function indexMetadataOf(indexName, entry, partitionCount) {
15275
15856
  }
15276
15857
 
15277
15858
  // src/engine/durability-integration.ts
15278
- function buildMetadata(indexName, hooks, documentCount) {
15859
+ function buildMetadata(indexName, hooks, documentCount, savedBytes) {
15279
15860
  const config = hooks.getIndexConfig(indexName);
15280
15861
  if (config === void 0) {
15281
15862
  return void 0;
@@ -15285,7 +15866,8 @@ function buildMetadata(indexName, hooks, documentCount) {
15285
15866
  indexName,
15286
15867
  config,
15287
15868
  manager === void 0 ? void 0 : metadataPartitionCount(manager),
15288
- documentCount
15869
+ documentCount,
15870
+ savedBytes
15289
15871
  );
15290
15872
  }
15291
15873
  function createDurabilityIntegration(tier, hooks) {
@@ -15293,7 +15875,7 @@ function createDurabilityIntegration(tier, hooks) {
15293
15875
  getManager: hooks.getManager,
15294
15876
  getVectorFieldPaths: hooks.getVectorFieldPaths,
15295
15877
  getVectorIndexes: hooks.getVectorIndexes,
15296
- buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
15878
+ buildMetadata: (indexName, documentCount, savedBytes) => buildMetadata(indexName, hooks, documentCount, savedBytes),
15297
15879
  buildTermsMetadata(indexName) {
15298
15880
  const config = hooks.getTermsConfig(indexName);
15299
15881
  return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
@@ -15308,7 +15890,8 @@ function createDurabilityIntegration(tier, hooks) {
15308
15890
  onCheckpointHeld: hooks.onCheckpointHeld,
15309
15891
  onFatalError: hooks.onFatalError,
15310
15892
  ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
15311
- ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
15893
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved },
15894
+ linkStoredSizes: hooks.linkStoredSizes
15312
15895
  };
15313
15896
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
15314
15897
  async function recordMutation(indexName, docId, operation, document, apply) {
@@ -15429,7 +16012,7 @@ var OLD_REGULAR_FILE = 0;
15429
16012
  var PAX_EXTENDED_HEADER = 120;
15430
16013
  var PAX_HEADER_NAME = "PaxHeader";
15431
16014
  var encoder3 = new TextEncoder();
15432
- var decoder3 = new TextDecoder("utf-8", { fatal: true });
16015
+ var decoder4 = new TextDecoder("utf-8", { fatal: true });
15433
16016
  function writeString(block, offset, length, value) {
15434
16017
  const bytes = encoder3.encode(value);
15435
16018
  block.set(bytes.subarray(0, length), offset);
@@ -15507,7 +16090,7 @@ function readString(block, offset, length) {
15507
16090
  let end = offset;
15508
16091
  while (end < offset + length && block[end] !== 0) end++;
15509
16092
  try {
15510
- return decoder3.decode(block.subarray(offset, end));
16093
+ return decoder4.decode(block.subarray(offset, end));
15511
16094
  } catch {
15512
16095
  invalidArchive("an entry header holds a name that is not UTF-8");
15513
16096
  }
@@ -15544,7 +16127,7 @@ var EQUALS = 61;
15544
16127
  var NEWLINE = 10;
15545
16128
  function decodeRecordText(bytes) {
15546
16129
  try {
15547
- return decoder3.decode(bytes);
16130
+ return decoder4.decode(bytes);
15548
16131
  } catch {
15549
16132
  invalidArchive("a pax extended header is not UTF-8");
15550
16133
  }
@@ -16232,7 +16815,7 @@ function createInvalidationIntegration(deps) {
16232
16815
 
16233
16816
  // src/engine/durability-wiring.ts
16234
16817
  function createDurabilityFromTier(tier, wiring) {
16235
- return createDurabilityIntegration(tier, {
16818
+ const integration = createDurabilityIntegration(tier, {
16236
16819
  checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
16237
16820
  getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
16238
16821
  getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
@@ -16252,8 +16835,25 @@ function createDurabilityFromTier(tier, wiring) {
16252
16835
  },
16253
16836
  queueDiskAdoption: wiring.queueDiskAdoption,
16254
16837
  awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
16255
- createIndexFromMetadata: wiring.createIndexFromMetadata,
16256
- recordCheckpoint: wiring.recordCheckpoint,
16838
+ async createIndexFromMetadata(metadata, loadData) {
16839
+ await wiring.createIndexFromMetadata(metadata, loadData);
16840
+ const entry = wiring.indexRegistry.get(metadata.indexName);
16841
+ if (entry === void 0) return;
16842
+ entry.storedSizes = integration.manager.storedSizesOf(metadata.indexName);
16843
+ entry.storedSizes.recordSavedBytes(entry.savedBytes);
16844
+ },
16845
+ recordCheckpoint(indexName, documentCount, partitionCount, savedBytes) {
16846
+ const entry = wiring.indexRegistry.get(indexName);
16847
+ if (entry === void 0) return;
16848
+ entry.documentCount = documentCount;
16849
+ entry.partitionCount = partitionCount;
16850
+ entry.savedBytes = savedBytes;
16851
+ entry.storedSizes?.recordSavedBytes(savedBytes);
16852
+ },
16853
+ linkStoredSizes(indexName, sizes) {
16854
+ const entry = wiring.indexRegistry.get(indexName);
16855
+ if (entry !== void 0) entry.storedSizes = sizes;
16856
+ },
16257
16857
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
16258
16858
  onCheckpointHeld: wiring.emitCheckpointHeld,
16259
16859
  onFatalError: wiring.emitFatalError,
@@ -16267,6 +16867,7 @@ function createDurabilityFromTier(tier, wiring) {
16267
16867
  }
16268
16868
  }
16269
16869
  });
16870
+ return integration;
16270
16871
  }
16271
16872
  async function reloadIndexAndCopies(orchestrator, durability, indexName) {
16272
16873
  const manager = durability?.manager;
@@ -16612,7 +17213,7 @@ function joined2(parts, total) {
16612
17213
  return bytes;
16613
17214
  }
16614
17215
  async function* exportLines(source, maxLineBytes) {
16615
- const decoder5 = new TextDecoder("utf-8", { fatal: true });
17216
+ const decoder6 = new TextDecoder("utf-8", { fatal: true });
16616
17217
  let parts = [];
16617
17218
  let pending = 0;
16618
17219
  let overflowing = false;
@@ -16629,7 +17230,7 @@ async function* exportLines(source, maxLineBytes) {
16629
17230
  parts = [];
16630
17231
  pending = 0;
16631
17232
  try {
16632
- return { line, text: decoder5.decode(bytes) };
17233
+ return { line, text: decoder6.decode(bytes) };
16633
17234
  } catch {
16634
17235
  return { line, refused: "not-utf8" };
16635
17236
  }
@@ -17164,10 +17765,10 @@ async function readSegment(checkpoint, ref) {
17164
17765
  });
17165
17766
  return decodeKeywordSegment(payload, false);
17166
17767
  }
17167
- var decoder4 = new TextDecoder();
17768
+ var decoder5 = new TextDecoder();
17168
17769
  function idAt(contents, ordinal) {
17169
17770
  const ids = contents.arrays.idTable;
17170
- return decoder4.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
17771
+ return decoder5.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
17171
17772
  }
17172
17773
  async function winnersOf(checkpoint, partition) {
17173
17774
  const winners = /* @__PURE__ */ new Map();
@@ -17688,8 +18289,9 @@ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
17688
18289
  partitionCount ?? void 0
17689
18290
  );
17690
18291
  }
17691
- function restoredRegistryEntry(prepared, manager, deps) {
18292
+ function restoredRegistryEntry(indexName, prepared, manager, deps) {
17692
18293
  return {
18294
+ storedSizes: deps.durability?.manager.storedSizesOf(indexName),
17693
18295
  config: prepared.config,
17694
18296
  language: prepared.language,
17695
18297
  embeddingAdapter: prepared.embeddingAdapter,
@@ -17789,13 +18391,17 @@ async function commitToStorage(indexName, restored, previous, deps) {
17789
18391
  for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
17790
18392
  await staging.commit();
17791
18393
  } catch (error) {
17792
- if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
18394
+ if (previousPartitions > 0) {
18395
+ durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
18396
+ await durable.measureRestoredStorage?.(indexName).catch(() => void 0);
18397
+ }
17793
18398
  throw error;
17794
18399
  }
17795
18400
  for (const paged of pagedSegmentsOf(previous)) paged.close();
17796
18401
  const restoredSegments = pagedSegmentsOf(restored.detached.manager);
17797
18402
  for (const paged of restoredSegments) paged.suspend();
17798
18403
  await staging.finish();
18404
+ await durable.measureRestoredStorage?.(indexName, restored.saved ?? void 0).catch(() => void 0);
17799
18405
  const relocate = await staging.relocation().catch(() => null);
17800
18406
  for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
17801
18407
  if (relocate === null) return;
@@ -17804,6 +18410,7 @@ async function commitToStorage(indexName, restored, previous, deps) {
17804
18410
  function activate(indexName, restored, entry, deps) {
17805
18411
  deps.executor.installDetachedIndex(restored.detached);
17806
18412
  deps.indexRegistry.set(indexName, entry);
18413
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
17807
18414
  restored.detached.attach();
17808
18415
  deps.clearAnalysisStale(indexName);
17809
18416
  const { prepared } = restored;
@@ -17843,6 +18450,7 @@ async function createFresh(indexName, restored, entry, deps) {
17843
18450
  deps.copies.desyncIndex(indexName);
17844
18451
  deps.executor.installDetachedIndex(restored.detached);
17845
18452
  deps.indexRegistry.set(indexName, entry);
18453
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
17846
18454
  try {
17847
18455
  await deps.indexState.registerReplacing(indexName, async () => {
17848
18456
  await commitToStorage(indexName, restored, void 0, deps);
@@ -17861,9 +18469,10 @@ async function stageMetadata(indexName, staging, entry) {
17861
18469
  await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
17862
18470
  }
17863
18471
  async function installRestoredIndex(indexName, restored, deps, history = null) {
17864
- const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
18472
+ const entry = restoredRegistryEntry(indexName, restored.prepared, restored.detached.manager, deps);
17865
18473
  const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
17866
18474
  if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
18475
+ if (restored.saved !== null) entry.savedBytes = restored.saved.bytes;
17867
18476
  let replaced;
17868
18477
  try {
17869
18478
  if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
@@ -17996,7 +18605,7 @@ async function loadStagedIndex(indexName, staged, staging, store, deps) {
17996
18605
  deps.executor.discardDetachedIndex(detached);
17997
18606
  throw error;
17998
18607
  }
17999
- return { prepared, detached, staging, checkpoint: manifest.checkpoint };
18608
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint, saved: savedBytesOfManifest(manifest) };
18000
18609
  }
18001
18610
  async function restoreArchive(indexName, reader, deps) {
18002
18611
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
@@ -18097,7 +18706,7 @@ function insertSnapshotDocuments(envelope, manager) {
18097
18706
  for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
18098
18707
  }
18099
18708
  async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
18100
- const entry = restoredRegistryEntry(prepared, built.manager, deps);
18709
+ const entry = restoredRegistryEntry(indexName, prepared, built.manager, deps);
18101
18710
  const archive = await buildArchiveFromMemory({
18102
18711
  indexName,
18103
18712
  metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
@@ -18121,7 +18730,7 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
18121
18730
  let manifest;
18122
18731
  try {
18123
18732
  insertSnapshotDocuments(envelope, built.manager);
18124
- if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
18733
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [], saved: null };
18125
18734
  manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
18126
18735
  } catch (err) {
18127
18736
  deps.executor.discardDetachedIndex(built);