@delali/narsil 0.3.1-next.181 → 0.3.1-next.184

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-CPSHS46N.mjs} +139 -177
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{embed-24IPR4LP.mjs → embed-GG6PS3EA.mjs} +43 -1
  4. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-ZTBHYQMN.mjs} +621 -314
  5. package/dist/browser/{resolve-vector-text-EJ6DHFF5.mjs → resolve-vector-text-XRR2OVHE.mjs} +19 -2
  6. package/dist/browser/{restore-queue-S4BWOGMU.mjs → restore-queue-SG5O735W.mjs} +7 -0
  7. package/dist/browser/{runtime-NU6VOVH2.mjs → runtime-OMKE7DF4.mjs} +23 -3
  8. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-4XMCITJ6.mjs} +886 -259
  9. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  10. package/dist/{chunk-KHJPMZE2.mjs → chunk-2KCCQK4R.mjs} +1 -1
  11. package/dist/{chunk-2KWTQ2P7.mjs → chunk-2YWWHMVC.mjs} +2 -2
  12. package/dist/{chunk-Y27E2MGC.mjs → chunk-3D4IMFSE.mjs} +2 -2
  13. package/dist/chunk-3ECIRFS5.mjs +61 -0
  14. package/dist/{chunk-XFOL77WH.mjs → chunk-4JBTALUC.mjs} +1 -1
  15. package/dist/{chunk-ISJ35523.mjs → chunk-5BUEBIXR.mjs} +2 -2
  16. package/dist/{chunk-DG6WIEB3.mjs → chunk-6SZQETSC.mjs} +1 -1
  17. package/dist/{chunk-UISV46WX.mjs → chunk-7WXGUTWK.mjs} +3 -3
  18. package/dist/{chunk-QFJMRB7D.mjs → chunk-A3OAFUZI.mjs} +1 -1
  19. package/dist/{chunk-GO6TRSJ6.mjs → chunk-BVZWWMOU.mjs} +1 -1
  20. package/dist/{chunk-CQX5B6L2.mjs → chunk-CKEOTSF5.mjs} +2 -2
  21. package/dist/{chunk-XQQKETAF.mjs → chunk-CODGHJVW.mjs} +4 -4
  22. package/dist/{chunk-34IOOYXA.mjs → chunk-CYCVPTTR.mjs} +2 -2
  23. package/dist/{chunk-26E5MO3Y.mjs → chunk-DU23J5JG.mjs} +5 -5
  24. package/dist/{chunk-JCWC75NH.mjs → chunk-EDEE7ETQ.mjs} +1 -1
  25. package/dist/{chunk-KL6W73BZ.mjs → chunk-EWGIF6FQ.mjs} +18 -1
  26. package/dist/{chunk-LPRRFCVU.mjs → chunk-F4OOBPKA.mjs} +2 -2
  27. package/dist/{chunk-BIIROQ5Q.mjs → chunk-FCQMZDRI.mjs} +2 -2
  28. package/dist/{chunk-YXZDTKQS.mjs → chunk-G3XRHAJA.mjs} +2 -2
  29. package/dist/{chunk-XTSFRWAS.mjs → chunk-GGIW6INA.mjs} +7 -6
  30. package/dist/{chunk-R7PLSVK3.mjs → chunk-H4QYQ5DE.mjs} +1 -1
  31. package/dist/{chunk-A4XYDUVN.mjs → chunk-HQPK73EW.mjs} +3 -3
  32. package/dist/{chunk-YPV2L7M5.mjs → chunk-II6XF6M4.mjs} +5 -5
  33. package/dist/{chunk-F3FDHXCJ.mjs → chunk-LR6JJVML.mjs} +251 -50
  34. package/dist/{chunk-ZB5X5WAK.mjs → chunk-M5I64AQW.mjs} +11 -6
  35. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  36. package/dist/{chunk-X4UMDSB4.mjs → chunk-N3M4NPML.mjs} +1 -1
  37. package/dist/{chunk-XITVDRRH.mjs → chunk-O337MDRJ.mjs} +6 -6
  38. package/dist/{chunk-XNFBYAXI.mjs → chunk-SEM3PBRH.mjs} +1 -1
  39. package/dist/{chunk-2C3BXZGA.mjs → chunk-SL53DHVE.mjs} +814 -539
  40. package/dist/{chunk-JP77OWMI.mjs → chunk-UPDNNOTG.mjs} +1 -60
  41. package/dist/{chunk-LOQHA6QK.mjs → chunk-URVH6BLO.mjs} +2 -2
  42. package/dist/{chunk-JDIATDPY.mjs → chunk-VYSWVQCE.mjs} +1 -1
  43. package/dist/{chunk-TNEU32EN.mjs → chunk-WLFOLAQI.mjs} +2 -1
  44. package/dist/{chunk-7FC2ZILY.mjs → chunk-WSIU6YXQ.mjs} +67 -78
  45. package/dist/{chunk-TBOZHYCF.mjs → chunk-Y7QUFIGV.mjs} +1 -1
  46. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  47. package/dist/client.d.ts +6 -6
  48. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  49. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  50. package/dist/{core-leaf-VIMO377O.mjs → core-leaf-NEUM7PU3.mjs} +3 -2
  51. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-37IH3ISS.mjs} +5 -4
  52. package/dist/distribution.d.ts +4 -4
  53. package/dist/distribution.mjs +398 -144
  54. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  55. package/dist/{embed-UNHDFGDU.mjs → embed-2QMYCD2Q.mjs} +43 -1
  56. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  57. package/dist/index.browser.mjs +1 -1
  58. package/dist/index.d.ts +4 -4
  59. package/dist/index.mjs +5 -4
  60. package/dist/index.react-native.mjs +4 -3
  61. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-PU2XT527.mjs} +15 -14
  62. package/dist/{module-5SXNUCD5.mjs → module-Z3CL4GJ4.mjs} +10 -9
  63. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-YIVJMQ4D.mjs} +9 -6
  64. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IZ6U7DVN.mjs} +15 -14
  65. package/dist/partial-words.browser.mjs +1 -1
  66. package/dist/partial-words.mjs +5 -4
  67. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  68. package/dist/persistence/durability/checkpoint-worker.mjs +28 -16
  69. package/dist/{query-actions-2JOLPELC.mjs → query-actions-7EOKT3WD.mjs} +5 -4
  70. package/dist/query-suggestions/merge-worker.mjs +8 -7
  71. package/dist/query-suggestions.browser.mjs +1 -1
  72. package/dist/query-suggestions.d.ts +1 -1
  73. package/dist/query-suggestions.mjs +6 -5
  74. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-7TPO4FLE.mjs} +2 -2
  75. package/dist/react-native/chunks/{chunk-J2BQTJJE.mjs → chunk-BN6FUB7N.mjs} +1 -1
  76. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-CBU4M3EU.mjs} +813 -538
  77. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-IEY4DARA.mjs} +1 -1
  78. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  79. package/dist/react-native/chunks/{chunk-I7DTUOK2.mjs → chunk-NLEFMPY3.mjs} +2 -60
  80. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-T5HF3FW3.mjs} +1 -1
  81. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-ZQPSXN6C.mjs} +1 -1
  82. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-ZUFOYZ5P.mjs} +1 -1
  83. package/dist/react-native/chunks/core-leaf-TPBRRWVN.mjs +7 -0
  84. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  85. package/dist/react-native/chunks/{embed-U7CIKEY4.mjs → embed-4LBSQD7U.mjs} +43 -1
  86. package/dist/react-native/chunks/{module-7BJ3VOWQ.mjs → module-COLNJNML.mjs} +4 -3
  87. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-QETIM6BI.mjs} +7 -4
  88. package/dist/react-native/chunks/{resolve-vector-text-FTPD5EY6.mjs → resolve-vector-text-FU7D4QP5.mjs} +18 -1
  89. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-3E2MJIJS.mjs} +14 -6
  90. package/dist/{runtime-4GKUIAKV.mjs → react-native/chunks/runtime-UOHD2XUJ.mjs} +23 -3
  91. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-IE4DXUZV.mjs} +5 -4
  92. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-62LQ5WZB.mjs} +6 -5
  93. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-LNOLKWZB.mjs} +895 -267
  94. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-Y7CBOPFZ.mjs} +6 -5
  95. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-PSUGL6MI.mjs} +6 -5
  96. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-MW5O5U3D.mjs} +5 -4
  97. package/dist/react-native/partial-words.mjs +4 -3
  98. package/dist/react-native/query-suggestions.mjs +4 -3
  99. package/dist/react-native/sync.mjs +4 -3
  100. package/dist/react-native/synonyms.mjs +4 -3
  101. package/dist/react.d.ts +7 -7
  102. package/dist/{rerank-E2HFKKGJ.mjs → rerank-KTV5L7MZ.mjs} +5 -4
  103. package/dist/resolve-vector-text-SD6BEMKZ.mjs +3 -0
  104. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-ES7TDJ5V.mjs} +15 -7
  105. package/dist/{react-native/chunks/runtime-BUCSAK4U.mjs → runtime-USQYKN3I.mjs} +23 -3
  106. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-DO75VKEV.mjs} +6 -5
  107. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-JGND6LG5.mjs} +6 -5
  108. package/dist/server.d.ts +8 -8
  109. package/dist/server.mjs +23 -22
  110. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-ZI3H2WJY.mjs} +595 -164
  111. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-YIH36Y5J.mjs} +7 -6
  112. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-FP5HSTFP.mjs} +9 -8
  113. package/dist/sync/filesystem.d.ts +2 -2
  114. package/dist/sync/filesystem.mjs +11 -10
  115. package/dist/sync/indexeddb.d.ts +2 -2
  116. package/dist/sync.browser.mjs +1 -1
  117. package/dist/sync.d.ts +3 -3
  118. package/dist/sync.mjs +5 -4
  119. package/dist/synonyms.browser.mjs +1 -1
  120. package/dist/synonyms.mjs +5 -4
  121. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  122. package/dist/{thread-OWXVP2ZA.mjs → thread-BJXE77AI.mjs} +14 -13
  123. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  124. package/dist/vector/search-worker.mjs +5 -4
  125. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-WYPRCNB7.mjs} +8 -7
  126. package/dist/workers/entry.mjs +11 -10
  127. package/openapi.json +32 -0
  128. package/package.json +7 -7
  129. package/dist/react-native/chunks/core-leaf-UDUMVC3M.mjs +0 -6
  130. package/dist/resolve-vector-text-6WPFFIX5.mjs +0 -2
@@ -1593,8 +1593,21 @@ function applyEntry(manager, entry, ctx) {
1593
1593
  });
1594
1594
  }
1595
1595
  }
1596
+ async function withinRebalanceHook(ctx, indexName, hook) {
1597
+ ctx.indexesInRebalanceHooks ??= /* @__PURE__ */ new Set();
1598
+ const running = ctx.indexesInRebalanceHooks;
1599
+ running.add(indexName);
1600
+ try {
1601
+ await hook();
1602
+ } finally {
1603
+ running.delete(indexName);
1604
+ }
1605
+ }
1596
1606
  async function afterEntry(manager, entry, ctx) {
1597
1607
  for (const field of entry.vectors.keys()) manager.getVectorIndexes().get(field)?.scheduleBuild();
1608
+ await withinRebalanceHook(ctx, entry.indexName, () => runEntryHooks(entry, ctx));
1609
+ }
1610
+ async function runEntryHooks(entry, ctx) {
1598
1611
  try {
1599
1612
  if (entry.action === "remove") {
1600
1613
  await ctx.pluginRegistry.runHook("afterRemove", { indexName: entry.indexName, docId: entry.docId });
@@ -1713,15 +1726,20 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
1713
1726
  },
1714
1727
  spill
1715
1728
  );
1716
- try {
1717
- await ctx.pluginRegistry.runHook("onPartitionSplit", {
1718
- indexName,
1719
- oldPartitionCount: oldCount,
1720
- newPartitionCount: targetPartitionCount
1721
- });
1722
- } catch (hookErr) {
1723
- console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
1724
- }
1729
+ await withinRebalanceHook(ctx, indexName, async () => {
1730
+ try {
1731
+ await ctx.pluginRegistry.runHook("onPartitionSplit", {
1732
+ indexName,
1733
+ oldPartitionCount: oldCount,
1734
+ newPartitionCount: targetPartitionCount
1735
+ });
1736
+ } catch (hookErr) {
1737
+ console.warn(
1738
+ "onPartitionSplit plugin hook failed:",
1739
+ hookErr instanceof Error ? hookErr.message : String(hookErr)
1740
+ );
1741
+ }
1742
+ });
1725
1743
  if (registry === void 0) await replayQueued(manager, waq, ctx);
1726
1744
  else {
1727
1745
  const hooks = [];
@@ -2109,6 +2127,8 @@ var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
2109
2127
  var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
2110
2128
  var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
2111
2129
  var LOG_RECORDS_PER_CHECKPOINT = 1e5;
2130
+ var REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE = 0.2;
2131
+ var DEFAULT_REWRITE_BYTES_PER_CHECKPOINT = 268435456;
2112
2132
  var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
2113
2133
  var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
2114
2134
  var CHECKPOINT_WORKER_IDLE_MS = 5e3;
@@ -2691,6 +2711,9 @@ function metadataToWire(meta) {
2691
2711
  if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
2692
2712
  wire.document_count = meta.documentCount;
2693
2713
  }
2714
+ if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
2715
+ wire.saved_bytes = meta.savedBytes;
2716
+ }
2694
2717
  if (meta.embedding) {
2695
2718
  wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
2696
2719
  }
@@ -2806,6 +2829,9 @@ function wireToMetadata(raw) {
2806
2829
  if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
2807
2830
  meta.documentCount = raw.document_count;
2808
2831
  }
2832
+ if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
2833
+ meta.savedBytes = raw.saved_bytes;
2834
+ }
2809
2835
  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
2810
2836
  meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
2811
2837
  }
@@ -4179,6 +4205,12 @@ function createReadCache(capacity) {
4179
4205
  function evictDownTo(limit) {
4180
4206
  for (const [key, entry] of entries) {
4181
4207
  if (held <= limit) return;
4208
+ if (entry.referenced) {
4209
+ entry.referenced = false;
4210
+ entries.delete(key);
4211
+ entries.set(key, entry);
4212
+ continue;
4213
+ }
4182
4214
  drop(key, entry);
4183
4215
  }
4184
4216
  }
@@ -4233,8 +4265,7 @@ function createReadCache(capacity) {
4233
4265
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
4234
4266
  const found = entries.get(key);
4235
4267
  if (found === void 0) return void 0;
4236
- entries.delete(key);
4237
- entries.set(key, found);
4268
+ found.referenced = true;
4238
4269
  return found.value;
4239
4270
  },
4240
4271
  put(fileNumber, kind, entry, value, bytes) {
@@ -4242,7 +4273,7 @@ function createReadCache(capacity) {
4242
4273
  const key = fileNumber * FILE_NUMBER_SCALE + entry * ENTRY_NUMBER_SCALE + kind;
4243
4274
  const existing = entries.get(key);
4244
4275
  if (existing !== void 0) drop(key, existing);
4245
- entries.set(key, { value, bytes, fileNumber });
4276
+ entries.set(key, { value, bytes, fileNumber, referenced: true });
4246
4277
  held += bytes;
4247
4278
  perFile.set(fileNumber, (perFile.get(fileNumber) ?? 0) + bytes);
4248
4279
  evictDownTo(capacity - reserved);
@@ -5542,28 +5573,55 @@ async function pagedListingOf(directory, ref) {
5542
5573
  function yieldToEventLoop2() {
5543
5574
  return new Promise((resolve) => setTimeout(resolve, 0));
5544
5575
  }
5576
+ function newerTombstonesOf(prepared, at) {
5577
+ const docIds = /* @__PURE__ */ new Set();
5578
+ for (let newer = at + 1; newer < prepared.length; newer++) {
5579
+ const access = prepared[newer].segment.paged;
5580
+ if (access === null) continue;
5581
+ for (const docId of access.snapshot.tombstones) docIds.add(docId);
5582
+ }
5583
+ return docIds;
5584
+ }
5585
+ function scansEveryId(segment, tombstoned) {
5586
+ const held = segment.arrays.idTable.count;
5587
+ return tombstoned.size * Math.ceil(Math.log2(held + 1)) > held;
5588
+ }
5589
+ function tombstoneScannedOrdinal(segment, tombstoned, ordinal) {
5590
+ const docId = segment.arrays.idTable.idAt(ordinal);
5591
+ if (tombstoned.has(docId)) segment.prepareTombstone(docId, void 0, void 0, ordinal)?.publish();
5592
+ }
5545
5593
  function applyNewerTombstones(prepared) {
5546
5594
  for (let at = 0; at < prepared.length; at++) {
5547
5595
  if (!prepared[at].isNew) continue;
5548
- for (let newer = at + 1; newer < prepared.length; newer++) {
5549
- const access = prepared[newer].segment.paged;
5550
- if (access === null) continue;
5551
- for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
5596
+ const { segment } = prepared[at];
5597
+ const tombstoned = newerTombstonesOf(prepared, at);
5598
+ if (!scansEveryId(segment, tombstoned)) {
5599
+ for (const docId of tombstoned) segment.tombstoneDocument(docId);
5600
+ continue;
5552
5601
  }
5602
+ const held = segment.arrays.idTable.count;
5603
+ for (let ordinal = 0; ordinal < held; ordinal++) tombstoneScannedOrdinal(segment, tombstoned, ordinal);
5553
5604
  }
5554
5605
  }
5555
5606
  async function applyNewerTombstonesInTurns(prepared) {
5556
- let applied = 0;
5607
+ let steps = 0;
5557
5608
  for (let at = 0; at < prepared.length; at++) {
5558
5609
  if (!prepared[at].isNew) continue;
5559
- for (let newer = at + 1; newer < prepared.length; newer++) {
5560
- const access = prepared[newer].segment.paged;
5561
- if (access === null) continue;
5562
- for (const docId of access.snapshot.tombstones) {
5563
- prepared[at].segment.tombstoneDocument(docId);
5564
- applied += 1;
5565
- if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5610
+ const { segment } = prepared[at];
5611
+ const tombstoned = newerTombstonesOf(prepared, at);
5612
+ if (!scansEveryId(segment, tombstoned)) {
5613
+ for (const docId of tombstoned) {
5614
+ segment.tombstoneDocument(docId);
5615
+ steps += 1;
5616
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5566
5617
  }
5618
+ continue;
5619
+ }
5620
+ const held = segment.arrays.idTable.count;
5621
+ for (let ordinal = 0; ordinal < held; ordinal++) {
5622
+ tombstoneScannedOrdinal(segment, tombstoned, ordinal);
5623
+ steps += 1;
5624
+ if (steps % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop2();
5567
5625
  }
5568
5626
  }
5569
5627
  }
@@ -8237,6 +8295,9 @@ function wholePartitionsWhereMostChanged(manager, rules) {
8237
8295
  return whole;
8238
8296
  };
8239
8297
  }
8298
+ function liveDocumentsOf(manager, targets) {
8299
+ return targets.map(({ partitionId }) => ({ partitionId, documentCount: manager.getPartition(partitionId).count() }));
8300
+ }
8240
8301
  async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
8241
8302
  const whileNoBuildPassOrMutationRuns = (capture3) => whileGraphsStayComplete(vectorIndexes, () => whileNoMutationApplies([...indexState.partitions.values()], capture3));
8242
8303
  return whileNoBuildPassOrMutationRuns(
@@ -8279,7 +8340,8 @@ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartit
8279
8340
  vectorPlans,
8280
8341
  wholePartitions,
8281
8342
  recordsLeftInLog,
8282
- unreplayedCutoffs: cutoffs
8343
+ unreplayedCutoffs: cutoffs,
8344
+ liveDocuments: memoryMatchesTheTargets && cutoffs.size === 0 ? liveDocumentsOf(manager, targets) : null
8283
8345
  };
8284
8346
  } catch (error) {
8285
8347
  for (const plan of vectorPlans.values()) plan.release();
@@ -8542,6 +8604,25 @@ async function pagedTombstonesFor(directory, older, written, deleted) {
8542
8604
  return [...tombstones];
8543
8605
  }
8544
8606
 
8607
+ // src/persistence/durability/stored-bytes.ts
8608
+ var NOTHING_SAVED = { bytes: 0, vectorBytes: 0, partitions: [] };
8609
+ function savedBytesOfManifest(manifest) {
8610
+ let bytes = 0;
8611
+ const partitions = [];
8612
+ for (const partition of manifest.partitions) {
8613
+ let partitionBytes = 0;
8614
+ for (const segment of partition.segments) partitionBytes += segment.bytes;
8615
+ partitions.push({ partitionId: partition.partitionId, bytes: partitionBytes });
8616
+ bytes += partitionBytes;
8617
+ }
8618
+ let vectorBytes = 0;
8619
+ for (const field of manifest.vectors) {
8620
+ for (const file of field.files) vectorBytes += file.bytes;
8621
+ for (const part of field.graphParts) vectorBytes += part.bytes;
8622
+ }
8623
+ return { bytes: bytes + vectorBytes, vectorBytes, partitions };
8624
+ }
8625
+
8545
8626
  // src/persistence/durability/segment/build-segment.ts
8546
8627
  function writeRecordsOf(documents, written) {
8547
8628
  const seqNos = new Float64Array(documents.length);
@@ -10487,6 +10568,9 @@ async function mergePagedWindow(input) {
10487
10568
  for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
10488
10569
  }
10489
10570
  }
10571
+ for (const docId of input.replacedAfterWindow ?? []) {
10572
+ for (const source of sources) source.segment.tombstoneDocument(docId);
10573
+ }
10490
10574
  const tombstones = /* @__PURE__ */ new Set();
10491
10575
  if (!input.dropsEveryTombstone) {
10492
10576
  for (const source of sources) {
@@ -10580,49 +10664,34 @@ function retainedTombstones(frozen, contents) {
10580
10664
  }
10581
10665
  return [...retained];
10582
10666
  }
10583
- async function compactPartitionSegments(input) {
10584
- const { segments, compactionThreshold } = input;
10585
- if (segments.length <= compactionThreshold) {
10586
- return { segments, nextSegmentId: input.nextSegmentId };
10587
- }
10588
- const paged = input.paged === true;
10589
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
10590
- if (window === null) {
10591
- return { segments, nextSegmentId: input.nextSegmentId };
10592
- }
10593
- const windowRefs = segments.slice(window.start, window.start + window.windowLength);
10594
- const replacing = (written2) => ({
10595
- segments: [
10596
- ...segments.slice(0, window.start),
10597
- ...written2.segments,
10598
- ...segments.slice(window.start + window.windowLength)
10599
- ],
10600
- nextSegmentId: written2.nextSegmentId
10601
- });
10602
- if (paged) {
10667
+ async function mergeSegmentWindow(input, window) {
10668
+ if (input.paged === true) {
10603
10669
  const { directory } = input.store;
10604
- if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
10605
- return replacing(
10606
- await mergePagedWindow({
10607
- store: input.store,
10608
- directory,
10609
- indexName: input.indexName,
10610
- partitionId: input.partitionId,
10611
- schema: input.schema,
10612
- partialWordAnalysis: input.partialWordAnalysis,
10613
- textAnalysis: input.textAnalysis,
10614
- nextSegmentId: input.nextSegmentId,
10615
- refs: windowRefs,
10616
- dropsEveryTombstone: window.start === 0
10617
- })
10618
- );
10670
+ if (directory === void 0) return null;
10671
+ return mergePagedWindow({
10672
+ store: input.store,
10673
+ directory,
10674
+ indexName: input.indexName,
10675
+ partitionId: input.partitionId,
10676
+ schema: input.schema,
10677
+ partialWordAnalysis: input.partialWordAnalysis,
10678
+ textAnalysis: input.textAnalysis,
10679
+ nextSegmentId: input.nextSegmentId,
10680
+ refs: window.refs,
10681
+ dropsEveryTombstone: window.startsTheList,
10682
+ ...window.replacedAfterWindow === void 0 ? {} : { replacedAfterWindow: window.replacedAfterWindow }
10683
+ });
10619
10684
  }
10620
10685
  const contents = [];
10621
- for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
10686
+ for (const ref of window.refs) contents.push(await readSegmentFile(input.store, ref));
10622
10687
  const frozen = contents.map(frozenOf);
10623
10688
  applyTimeOrder(frozen, contents);
10689
+ for (const docId of window.replacedAfterWindow ?? []) {
10690
+ for (const segment of frozen) segment.tombstoneDocument(docId);
10691
+ }
10624
10692
  const writeRecords = writeRecordsOf2(frozen, contents);
10625
- const written = await writeSegmentsOf({
10693
+ const keepsTombstones = !(window.startsTheList && window.replacedAfterWindow !== void 0);
10694
+ return writeSegmentsOf({
10626
10695
  store: input.store,
10627
10696
  indexName: input.indexName,
10628
10697
  partitionId: input.partitionId,
@@ -10631,11 +10700,33 @@ async function compactPartitionSegments(input) {
10631
10700
  partialWordAnalysis: input.partialWordAnalysis,
10632
10701
  textAnalysis: input.textAnalysis,
10633
10702
  selections: frozen.map((segment) => ({ segment })),
10634
- tombstones: retainedTombstones(frozen, contents),
10703
+ tombstones: keepsTombstones ? retainedTombstones(frozen, contents) : [],
10635
10704
  mergesIntoOneFile: true,
10636
10705
  ...writeRecords === void 0 ? {} : { writeRecords }
10637
10706
  });
10638
- return replacing(written);
10707
+ }
10708
+ async function compactPartitionSegments(input) {
10709
+ const { segments, compactionThreshold } = input;
10710
+ if (segments.length <= compactionThreshold) {
10711
+ return { segments, nextSegmentId: input.nextSegmentId };
10712
+ }
10713
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, input.paged === true);
10714
+ if (window === null) {
10715
+ return { segments, nextSegmentId: input.nextSegmentId };
10716
+ }
10717
+ const written = await mergeSegmentWindow(input, {
10718
+ refs: segments.slice(window.start, window.start + window.windowLength),
10719
+ startsTheList: window.start === 0
10720
+ });
10721
+ if (written === null) return { segments, nextSegmentId: input.nextSegmentId };
10722
+ return {
10723
+ segments: [
10724
+ ...segments.slice(0, window.start),
10725
+ ...written.segments,
10726
+ ...segments.slice(window.start + window.windowLength)
10727
+ ],
10728
+ nextSegmentId: written.nextSegmentId
10729
+ };
10639
10730
  }
10640
10731
 
10641
10732
  // src/persistence/durability/segment/manifest-encode.ts
@@ -10684,6 +10775,161 @@ function encodeSegmentManifest(manifest) {
10684
10775
  return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
10685
10776
  }
10686
10777
 
10778
+ // src/persistence/paged/segment-scan.ts
10779
+ async function openPagedSegmentScan(directory, ref) {
10780
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
10781
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10782
+ const segment = createPagedFrozenSegment(snapshot, cache);
10783
+ const { paged } = segment;
10784
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10785
+ const ids = segment.arrays.idTable;
10786
+ const tombstones = new Set(snapshot.tombstones);
10787
+ return {
10788
+ count: snapshot.documentCount,
10789
+ schema: snapshot.schema,
10790
+ tombstones,
10791
+ idAt: (ordinal) => ids.idAt(ordinal),
10792
+ holds: (docId) => ids.ordinalOf(docId) >= 0,
10793
+ storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
10794
+ close: () => {
10795
+ paged.close();
10796
+ cache.clear();
10797
+ }
10798
+ };
10799
+ }
10800
+ async function checkPagedSegmentFully(directory, ref) {
10801
+ const listing = await pagedListingOf(directory, ref);
10802
+ await requirePagedPayloadChecksum(directory, listing);
10803
+ const snapshot = openPagedSnapshot(listing);
10804
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
10805
+ const segment = createPagedFrozenSegment(snapshot, cache);
10806
+ const { paged } = segment;
10807
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
10808
+ try {
10809
+ await checkPagedContents(segment, cache);
10810
+ return snapshot.schema;
10811
+ } finally {
10812
+ paged.close();
10813
+ cache.clear();
10814
+ }
10815
+ }
10816
+ async function bytesThatSegmentsHoldInMemory(directory, manifest) {
10817
+ let bytes = 0;
10818
+ for (const partition of manifest.partitions) {
10819
+ for (const ref of partition.segments) {
10820
+ const paged = await isPagedSegmentFile(directory, ref.key);
10821
+ bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
10822
+ }
10823
+ }
10824
+ return bytes;
10825
+ }
10826
+ function countsAsLatestHolder(scans, at, docId) {
10827
+ for (let later = at + 1; later < scans.length; later++) {
10828
+ if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
10829
+ }
10830
+ return true;
10831
+ }
10832
+
10833
+ // src/persistence/durability/segment/removed-documents.ts
10834
+ var decoder2 = new TextDecoder();
10835
+ async function readSegmentIds(store, ref) {
10836
+ const { directory } = store;
10837
+ if (directory !== void 0 && await isPagedSegmentFile(directory, ref.key)) {
10838
+ const scan = await openPagedSegmentScan(directory, ref);
10839
+ try {
10840
+ const ids2 = new Array(scan.count);
10841
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) ids2[ordinal] = scan.idAt(ordinal);
10842
+ return { ids: ids2, tombstones: [...scan.tombstones] };
10843
+ } finally {
10844
+ scan.close();
10845
+ }
10846
+ }
10847
+ const contents = await readSegmentFile(store, ref);
10848
+ const table = contents.arrays.idTable;
10849
+ const ids = new Array(contents.arrays.documentCount);
10850
+ for (let ordinal = 0; ordinal < ids.length; ordinal++) {
10851
+ ids[ordinal] = decoder2.decode(table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]));
10852
+ }
10853
+ return { ids, tombstones: contents.tombstones };
10854
+ }
10855
+ async function documentsReplacedLater(store, segments) {
10856
+ const later = /* @__PURE__ */ new Set();
10857
+ const replaced = new Array(segments.length);
10858
+ for (let at = segments.length - 1; at >= 0; at--) {
10859
+ const { ids, tombstones } = await readSegmentIds(store, segments[at]);
10860
+ const found = [];
10861
+ for (const docId of ids) if (later.has(docId)) found.push(docId);
10862
+ replaced[at] = found;
10863
+ if (at === 0) break;
10864
+ for (const docId of ids) later.add(docId);
10865
+ for (const docId of tombstones) later.add(docId);
10866
+ }
10867
+ return replaced;
10868
+ }
10869
+ function exceedsRemovedShare(removed, held) {
10870
+ return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
10871
+ }
10872
+ function freedBytesOf(ref, replacedCount) {
10873
+ return ref.docCount === 0 ? 0 : ref.bytes * replacedCount / ref.docCount;
10874
+ }
10875
+ function chooseRewrites(segments, replaced, budget) {
10876
+ let held = 0;
10877
+ let removed = 0;
10878
+ for (let at = 0; at < segments.length; at++) {
10879
+ held += segments[at].docCount;
10880
+ removed += replaced[at].length;
10881
+ }
10882
+ const candidates = [];
10883
+ for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
10884
+ candidates.sort(
10885
+ (left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
10886
+ );
10887
+ const chosen = [];
10888
+ for (const at of candidates) {
10889
+ if (!exceedsRemovedShare(removed, held)) break;
10890
+ const bytes = segments[at].bytes;
10891
+ if (chosen.length > 0 && budget.spentBytes + bytes > budget.limitBytes) continue;
10892
+ chosen.push(at);
10893
+ budget.spentBytes += bytes;
10894
+ held -= replaced[at].length;
10895
+ removed -= replaced[at].length;
10896
+ }
10897
+ return chosen.sort((left, right) => left - right);
10898
+ }
10899
+ function holdsSomething(ref) {
10900
+ return ref.docCount > 0 || ref.tombstoneCount > 0;
10901
+ }
10902
+ async function rewriteRemovedDocuments(input) {
10903
+ const { segments, liveDocumentCount } = input;
10904
+ const unchanged = { segments, nextSegmentId: input.nextSegmentId };
10905
+ if (liveDocumentCount === void 0) return unchanged;
10906
+ let held = 0;
10907
+ for (const ref of segments) held += ref.docCount;
10908
+ if (!exceedsRemovedShare(held - liveDocumentCount, held)) return unchanged;
10909
+ const replaced = await documentsReplacedLater(input.store, segments);
10910
+ const chosen = new Set(chooseRewrites(segments, replaced, input.budget));
10911
+ if (chosen.size === 0) return unchanged;
10912
+ const rewritten = [];
10913
+ let nextSegmentId = input.nextSegmentId;
10914
+ for (let at = 0; at < segments.length; at++) {
10915
+ if (!chosen.has(at)) {
10916
+ rewritten.push(segments[at]);
10917
+ continue;
10918
+ }
10919
+ const written = await mergeSegmentWindow(
10920
+ { ...input, nextSegmentId },
10921
+ { refs: [segments[at]], startsTheList: rewritten.length === 0, replacedAfterWindow: replaced[at] }
10922
+ );
10923
+ if (written === null) {
10924
+ rewritten.push(segments[at]);
10925
+ continue;
10926
+ }
10927
+ nextSegmentId = written.nextSegmentId;
10928
+ rewritten.push(...written.segments.filter(holdsSomething));
10929
+ }
10930
+ return { segments: rewritten, nextSegmentId };
10931
+ }
10932
+
10687
10933
  // src/persistence/durability/segment/write.ts
10688
10934
  function textAnalysisOf(config) {
10689
10935
  let resolved = null;
@@ -10806,8 +11052,21 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
10806
11052
  compactionThreshold: context.compactionThreshold,
10807
11053
  paged: context.input.metadata.textStorage === "disk"
10808
11054
  });
10809
- segments = compacted.segments;
10810
- nextSegmentId = compacted.nextSegmentId;
11055
+ const rewritten = await rewriteRemovedDocuments({
11056
+ store: context.input.store,
11057
+ indexName: context.indexName,
11058
+ partitionId,
11059
+ schema: context.input.metadata.schema,
11060
+ partialWordAnalysis: context.partialWordAnalysis,
11061
+ textAnalysis: context.textAnalysis,
11062
+ segments: compacted.segments,
11063
+ nextSegmentId: compacted.nextSegmentId,
11064
+ paged: context.input.metadata.textStorage === "disk",
11065
+ liveDocumentCount: context.input.liveDocuments?.find((entry) => entry.partitionId === partitionId)?.documentCount,
11066
+ budget: context.rewriteBudget
11067
+ });
11068
+ segments = rewritten.segments;
11069
+ nextSegmentId = rewritten.nextSegmentId;
10811
11070
  return { partitionId, nextSegmentId, segments };
10812
11071
  }
10813
11072
  function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
@@ -10832,7 +11091,11 @@ async function writeCheckpointSegments(input) {
10832
11091
  partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
10833
11092
  textAnalysis,
10834
11093
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
10835
- compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
11094
+ compactionThreshold: resolveCompactionThreshold(input.compactionThreshold),
11095
+ rewriteBudget: {
11096
+ spentBytes: 0,
11097
+ limitBytes: input.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT
11098
+ }
10836
11099
  };
10837
11100
  const checkpointByPartition = /* @__PURE__ */ new Map();
10838
11101
  for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
@@ -10869,6 +11132,7 @@ async function writeCheckpointSegments(input) {
10869
11132
  async function commitCheckpointManifest(store, metadata, written, vectors) {
10870
11133
  const indexName = metadata.indexName;
10871
11134
  const manifest = {
11135
+ version: SEGMENT_MANIFEST_VERSION,
10872
11136
  schema: metadata.schema,
10873
11137
  language: metadata.language,
10874
11138
  ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
@@ -10878,9 +11142,9 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
10878
11142
  };
10879
11143
  const packed = await encodeSegmentManifest(manifest);
10880
11144
  await store.write(manifestKey(indexName), packed.chunks);
10881
- return { garbage: await unreferencedKeys(store, indexName, manifest) };
11145
+ return { manifest, saved: savedBytesOfManifest(manifest) };
10882
11146
  }
10883
- async function unreferencedKeys(store, indexName, manifest) {
11147
+ async function checkpointGarbage(store, indexName, manifest) {
10884
11148
  const referenced = manifestReferencedKeys(manifest);
10885
11149
  const stored = await store.list(segmentsPrefix(indexName));
10886
11150
  return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
@@ -10902,7 +11166,8 @@ function workerCanWriteEverySegment(input) {
10902
11166
  }
10903
11167
  async function writeIndexCheckpointSegments(input) {
10904
11168
  const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
10905
- const { keepsWriteRecords } = input;
11169
+ const { keepsWriteRecords, liveDocuments, rewriteBytesPerCheckpoint } = input;
11170
+ const counted = liveDocuments === null ? {} : { liveDocuments, rewriteBytesPerCheckpoint };
10906
11171
  const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
10907
11172
  root: directory.root,
10908
11173
  metadata,
@@ -10910,7 +11175,8 @@ async function writeIndexCheckpointSegments(input) {
10910
11175
  compactionThreshold,
10911
11176
  prior,
10912
11177
  capturedPartitions: [...capturedPartitions],
10913
- keepsWriteRecords
11178
+ keepsWriteRecords,
11179
+ ...counted
10914
11180
  }) : null;
10915
11181
  if (offloaded !== null) {
10916
11182
  return offloaded;
@@ -10924,7 +11190,8 @@ async function writeIndexCheckpointSegments(input) {
10924
11190
  prior,
10925
11191
  wholePartitions,
10926
11192
  capturedPartitions,
10927
- keepsWriteRecords
11193
+ keepsWriteRecords,
11194
+ ...counted
10928
11195
  });
10929
11196
  }
10930
11197
 
@@ -11007,61 +11274,6 @@ async function startReadingOpeningSectionsAhead(directory, manifest) {
11007
11274
  }
11008
11275
  }
11009
11276
 
11010
- // src/persistence/paged/segment-scan.ts
11011
- async function openPagedSegmentScan(directory, ref) {
11012
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
11013
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
11014
- const segment = createPagedFrozenSegment(snapshot, cache);
11015
- const { paged } = segment;
11016
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
11017
- const ids = segment.arrays.idTable;
11018
- const tombstones = new Set(snapshot.tombstones);
11019
- return {
11020
- count: snapshot.documentCount,
11021
- schema: snapshot.schema,
11022
- tombstones,
11023
- idAt: (ordinal) => ids.idAt(ordinal),
11024
- holds: (docId) => ids.ordinalOf(docId) >= 0,
11025
- storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
11026
- close: () => {
11027
- paged.close();
11028
- cache.clear();
11029
- }
11030
- };
11031
- }
11032
- async function checkPagedSegmentFully(directory, ref) {
11033
- const listing = await pagedListingOf(directory, ref);
11034
- await requirePagedPayloadChecksum(directory, listing);
11035
- const snapshot = openPagedSnapshot(listing);
11036
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
11037
- const segment = createPagedFrozenSegment(snapshot, cache);
11038
- const { paged } = segment;
11039
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
11040
- try {
11041
- await checkPagedContents(segment, cache);
11042
- return snapshot.schema;
11043
- } finally {
11044
- paged.close();
11045
- cache.clear();
11046
- }
11047
- }
11048
- async function bytesThatSegmentsHoldInMemory(directory, manifest) {
11049
- let bytes = 0;
11050
- for (const partition of manifest.partitions) {
11051
- for (const ref of partition.segments) {
11052
- const paged = await isPagedSegmentFile(directory, ref.key);
11053
- bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
11054
- }
11055
- }
11056
- return bytes;
11057
- }
11058
- function countsAsLatestHolder(scans, at, docId) {
11059
- for (let later = at + 1; later < scans.length; later++) {
11060
- if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
11061
- }
11062
- return true;
11063
- }
11064
-
11065
11277
  // src/persistence/durability/segment/load-partition.ts
11066
11278
  function requireAgreedSchema(agreed, schema, key) {
11067
11279
  for (const [field, type] of Object.entries(schema)) {
@@ -11075,6 +11287,20 @@ function requireAgreedSchema(agreed, schema, key) {
11075
11287
  }
11076
11288
  }
11077
11289
  }
11290
+ function requireListedCounts(ref, documentCount, tombstoneCount) {
11291
+ if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
11292
+ throw new NarsilError(
11293
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
11294
+ `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
11295
+ {
11296
+ key: ref.key,
11297
+ documentCount,
11298
+ tombstoneCount,
11299
+ listedDocCount: ref.docCount,
11300
+ listedTombstoneCount: ref.tombstoneCount
11301
+ }
11302
+ );
11303
+ }
11078
11304
  function removeWhatTheSegmentReplaces(manager, segment, contents) {
11079
11305
  manager.beginBatchRemove();
11080
11306
  try {
@@ -11109,6 +11335,7 @@ function removeFromPartition(manager, partitionId, docIds) {
11109
11335
  async function attachPagedSegment(directory, ref, partitionId, manager, options) {
11110
11336
  if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
11111
11337
  const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
11338
+ requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
11112
11339
  requireAgreedSchema(options.schema, snapshot.schema, ref.key);
11113
11340
  const segment = createPagedFrozenSegment(snapshot);
11114
11341
  removeFromPartition(manager, partitionId, snapshot.tombstones);
@@ -11154,6 +11381,7 @@ async function attachPartitionSegments(store, partition, manager, options) {
11154
11381
  }
11155
11382
  }
11156
11383
  const contents = await readSegmentFile(store, ref);
11384
+ requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
11157
11385
  requireAgreedSchema(options.schema, contents.schema, ref.key);
11158
11386
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
11159
11387
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
@@ -11612,23 +11840,28 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11612
11840
  metadata,
11613
11841
  targets,
11614
11842
  compactionThreshold: input.compactionThreshold,
11843
+ rewriteBytesPerCheckpoint: input.rewriteBytesPerCheckpoint,
11615
11844
  canOffload: input.canOffload,
11616
11845
  prior: priorForLayout(priorManifest, capture3.partitionCount),
11617
11846
  wholePartitions: capture3.wholePartitions.contents,
11618
11847
  capturedPartitions: capture3.wholePartitions.captured,
11619
- keepsWriteRecords: writeRecords !== null
11848
+ keepsWriteRecords: writeRecords !== null,
11849
+ liveDocuments: capture3.liveDocuments
11620
11850
  })
11621
11851
  );
11622
11852
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
11623
11853
  if (aTargetFailed(input.indexState, targets)) return 0;
11624
11854
  const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
11855
+ input.recordSavedBytes(written.saved);
11625
11856
  input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
11626
11857
  recordWrittenVectors(vectorIndexes, liveVectors.written);
11627
11858
  await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
11628
11859
  const changes = changesSinceCapture();
11629
11860
  const droppedThisCheckpoint = changes !== null && capture3.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
11861
+ await writeCheckpointMetadata(input, documentCount, capture3.partitionCount, written.saved.bytes);
11630
11862
  if (!input.holdsCheckpointFiles()) {
11631
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
11863
+ const garbage = await checkpointGarbage(store, input.indexName, written.manifest);
11864
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, garbage);
11632
11865
  const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
11633
11866
  const keptForSync = await input.keepsCheckpointKey();
11634
11867
  await removeCheckpointGarbage(
@@ -11636,19 +11869,6 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11636
11869
  removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
11637
11870
  );
11638
11871
  }
11639
- const checkpointDocumentCount = documentCount;
11640
- await input.queueMetadataWrite(input.indexName, async () => {
11641
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
11642
- if (checkpointMetadata === void 0) {
11643
- return;
11644
- }
11645
- const bytes = await writeMetadataEnvelope(
11646
- { ...checkpointMetadata, partitionCount: capture3.partitionCount },
11647
- { checksum: true }
11648
- );
11649
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
11650
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture3.partitionCount);
11651
- });
11652
11872
  const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
11653
11873
  await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
11654
11874
  if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
@@ -11666,6 +11886,18 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
11666
11886
  const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
11667
11887
  return anotherIsDue ? eligibleBeyond : capture3.recordsLeftInLog;
11668
11888
  }
11889
+ async function writeCheckpointMetadata(input, documentCount, partitionCount, savedBytes) {
11890
+ const { indexName, hooks } = input;
11891
+ await input.queueMetadataWrite(indexName, async () => {
11892
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
11893
+ if (checkpointMetadata === void 0) {
11894
+ return;
11895
+ }
11896
+ const bytes = await writeMetadataEnvelope({ ...checkpointMetadata, partitionCount }, { checksum: true });
11897
+ await input.directory.atomicWrite(`${indexName}/meta`, bytes);
11898
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, savedBytes);
11899
+ });
11900
+ }
11669
11901
  function recordsBeyond(indexState, targets, cutoffs) {
11670
11902
  let records = 0;
11671
11903
  for (const [partitionId, partition] of indexState.partitions) {
@@ -11830,6 +12062,258 @@ function createHeldCheckpoints(hooks) {
11830
12062
  };
11831
12063
  }
11832
12064
 
12065
+ // src/persistence/durability/index-stored-sizes.ts
12066
+ function refreshStoredSizes(sizes) {
12067
+ const known = sizes.saved !== null || !sizes.measuring && sizes.held;
12068
+ sizes.known = known;
12069
+ if (known) {
12070
+ sizes.savedBytes = sizes.saved?.bytes ?? 0;
12071
+ sizes.vectorSavedBytes = sizes.saved?.vectorBytes ?? 0;
12072
+ sizes.logBytes = sizes.tracksLog ? sizes.logged?.bytes ?? 0 : void 0;
12073
+ return;
12074
+ }
12075
+ sizes.savedBytes = sizes.measuring ? void 0 : sizes.recorded;
12076
+ sizes.vectorSavedBytes = void 0;
12077
+ sizes.logBytes = void 0;
12078
+ }
12079
+ function savedBytesByPartitionOf(saved) {
12080
+ const byPartition = [];
12081
+ if (saved === null) return byPartition;
12082
+ for (const { partitionId, bytes } of saved.partitions) byPartition[partitionId] = bytes;
12083
+ return byPartition;
12084
+ }
12085
+ function createTrackedSizes(tracksLog) {
12086
+ const sizes = {
12087
+ savedBytes: void 0,
12088
+ logBytes: void 0,
12089
+ vectorSavedBytes: void 0,
12090
+ saved: null,
12091
+ savedBytesByPartition: [],
12092
+ measuring: false,
12093
+ held: false,
12094
+ recorded: void 0,
12095
+ logged: null,
12096
+ known: false,
12097
+ tracksLog,
12098
+ addTo(stats) {
12099
+ if (!sizes.known) return;
12100
+ const saved = sizes.savedBytesByPartition;
12101
+ const logged = sizes.tracksLog ? sizes.logged?.bytesByPartition ?? [] : null;
12102
+ for (const partition of stats) {
12103
+ partition.savedBytes = saved[partition.partitionId] ?? 0;
12104
+ if (logged !== null) partition.logBytes = logged[partition.partitionId] ?? 0;
12105
+ }
12106
+ },
12107
+ recordSavedBytes(bytes) {
12108
+ sizes.recorded = bytes;
12109
+ refreshStoredSizes(sizes);
12110
+ }
12111
+ };
12112
+ return sizes;
12113
+ }
12114
+ function createStoredSizesRecord(tracksLog) {
12115
+ const sizesByIndex = /* @__PURE__ */ new Map();
12116
+ function of(indexName) {
12117
+ let sizes = sizesByIndex.get(indexName);
12118
+ if (sizes === void 0) {
12119
+ sizes = createTrackedSizes(tracksLog);
12120
+ sizesByIndex.set(indexName, sizes);
12121
+ }
12122
+ return sizes;
12123
+ }
12124
+ function forget(indexName) {
12125
+ const sizes = sizesByIndex.get(indexName);
12126
+ if (sizes === void 0) return;
12127
+ sizes.saved = null;
12128
+ sizes.savedBytesByPartition = [];
12129
+ sizes.measuring = false;
12130
+ sizes.held = false;
12131
+ refreshStoredSizes(sizes);
12132
+ }
12133
+ return {
12134
+ of,
12135
+ setSaved(indexName, saved) {
12136
+ const sizes = of(indexName);
12137
+ sizes.saved = saved;
12138
+ sizes.savedBytesByPartition = savedBytesByPartitionOf(saved);
12139
+ refreshStoredSizes(sizes);
12140
+ },
12141
+ setMeasuring(indexName, measuring) {
12142
+ const sizes = of(indexName);
12143
+ sizes.measuring = measuring;
12144
+ refreshStoredSizes(sizes);
12145
+ },
12146
+ hold(indexName) {
12147
+ const sizes = of(indexName);
12148
+ if (!sizes.held) {
12149
+ sizes.held = true;
12150
+ refreshStoredSizes(sizes);
12151
+ }
12152
+ return sizes;
12153
+ },
12154
+ forget,
12155
+ drop(indexName) {
12156
+ forget(indexName);
12157
+ sizesByIndex.delete(indexName);
12158
+ }
12159
+ };
12160
+ }
12161
+
12162
+ // src/persistence/durability/log-bytes.ts
12163
+ var LOG_SEGMENT_KEY = /^(.+)\/wal\/(\d+)\/\d{16}$/;
12164
+ var HISTORY_SEGMENT_KEY = /^(.+)\/history\/(\d+)\/\d{16}-\d{16}$/;
12165
+ function loggedFileOf(key) {
12166
+ const match = LOG_SEGMENT_KEY.exec(key) ?? HISTORY_SEGMENT_KEY.exec(key);
12167
+ if (match === null) return null;
12168
+ const partitionId = Number.parseInt(match[2], 10);
12169
+ return Number.isSafeInteger(partitionId) ? { indexName: match[1], partitionId } : null;
12170
+ }
12171
+ function emptyLog() {
12172
+ return { files: /* @__PURE__ */ new Map(), bytes: 0, bytesByPartition: [], sizes: null };
12173
+ }
12174
+ function addBytes(log, partitionId, delta) {
12175
+ log.bytes += delta;
12176
+ log.bytesByPartition[partitionId] = (log.bytesByPartition[partitionId] ?? 0) + delta;
12177
+ if (log.sizes !== null) refreshStoredSizes(log.sizes);
12178
+ }
12179
+ function makeCurrent(previous, log, sizes) {
12180
+ if (previous !== void 0 && previous !== log) previous.sizes = null;
12181
+ log.sizes = sizes;
12182
+ sizes.logged = log;
12183
+ refreshStoredSizes(sizes);
12184
+ }
12185
+ function setFileBytes(log, key, partitionId, bytes) {
12186
+ const previous = log.files.get(key);
12187
+ if (previous !== void 0) addBytes(log, previous.partitionId, -previous.bytes);
12188
+ log.files.set(key, { partitionId, bytes });
12189
+ addBytes(log, partitionId, bytes);
12190
+ }
12191
+ function removeFile(log, key) {
12192
+ const previous = log.files.get(key);
12193
+ if (previous === void 0) return void 0;
12194
+ log.files.delete(key);
12195
+ addBytes(log, previous.partitionId, -previous.bytes);
12196
+ return previous;
12197
+ }
12198
+ function createLogBytesLedger(base, sizes) {
12199
+ const logs = /* @__PURE__ */ new Map();
12200
+ function logOf(indexName) {
12201
+ let log = logs.get(indexName);
12202
+ if (log === void 0) {
12203
+ log = emptyLog();
12204
+ logs.set(indexName, log);
12205
+ makeCurrent(void 0, log, sizes.of(indexName));
12206
+ }
12207
+ return log;
12208
+ }
12209
+ function track(key, bytes) {
12210
+ const file = loggedFileOf(key);
12211
+ if (file !== null) setFileBytes(logOf(file.indexName), key, file.partitionId, bytes);
12212
+ }
12213
+ function untrack(key) {
12214
+ const file = loggedFileOf(key);
12215
+ if (file === null) return void 0;
12216
+ const log = logs.get(file.indexName);
12217
+ return log === void 0 ? void 0 : removeFile(log, key);
12218
+ }
12219
+ async function currentLength(key) {
12220
+ return await base.byteLength?.(key) ?? (await base.read(key))?.byteLength ?? 0;
12221
+ }
12222
+ function trackedHandle(key, file, handle) {
12223
+ async function remeasure() {
12224
+ const log = logs.get(file.indexName);
12225
+ if (log?.files.has(key) === true) setFileBytes(log, key, file.partitionId, await handle.size());
12226
+ }
12227
+ return {
12228
+ async append(bytes) {
12229
+ try {
12230
+ await handle.append(bytes);
12231
+ } catch (error) {
12232
+ await remeasure().catch(() => void 0);
12233
+ throw error;
12234
+ }
12235
+ const log = logs.get(file.indexName);
12236
+ const entry = log?.files.get(key);
12237
+ if (log === void 0 || entry === void 0) return;
12238
+ entry.bytes += bytes.byteLength;
12239
+ addBytes(log, file.partitionId, bytes.byteLength);
12240
+ },
12241
+ sync: () => handle.sync(),
12242
+ async truncate(byteLength2) {
12243
+ try {
12244
+ await handle.truncate(byteLength2);
12245
+ } finally {
12246
+ await remeasure().catch(() => void 0);
12247
+ }
12248
+ },
12249
+ size: () => handle.size(),
12250
+ close: () => handle.close()
12251
+ };
12252
+ }
12253
+ const directory = {
12254
+ root: base.root,
12255
+ async appendHandle(key) {
12256
+ const handle = await base.appendHandle(key);
12257
+ const file = loggedFileOf(key);
12258
+ if (file === null) return handle;
12259
+ if (logs.get(file.indexName)?.files.has(key) !== true) track(key, await handle.size());
12260
+ return trackedHandle(key, file, handle);
12261
+ },
12262
+ markerHandle: (key) => base.markerHandle(key),
12263
+ async atomicWrite(key, data) {
12264
+ await base.atomicWrite(key, data);
12265
+ if (loggedFileOf(key) === null) return;
12266
+ let bytes = 0;
12267
+ for (const chunk of Array.isArray(data) ? data : [data]) bytes += chunk.byteLength;
12268
+ track(key, bytes);
12269
+ },
12270
+ syncDirectoryOf: (key) => base.syncDirectoryOf(key),
12271
+ read: (key) => base.read(key),
12272
+ async remove(key) {
12273
+ await base.remove(key);
12274
+ untrack(key);
12275
+ },
12276
+ async move(from, to) {
12277
+ await base.move(from, to);
12278
+ const moved = untrack(from);
12279
+ if (loggedFileOf(to) !== null) track(to, moved?.bytes ?? await currentLength(to));
12280
+ },
12281
+ list: (prefix) => base.list(prefix),
12282
+ pathOf: (key) => base.pathOf(key)
12283
+ };
12284
+ const { readRange, byteLength, identity, lock } = base;
12285
+ if (readRange !== void 0) directory.readRange = (key, offset, length) => readRange.call(base, key, offset, length);
12286
+ if (byteLength !== void 0) directory.byteLength = (key) => byteLength.call(base, key);
12287
+ if (identity !== void 0) directory.identity = () => identity.call(base);
12288
+ if (lock !== void 0) directory.lock = () => lock.call(base);
12289
+ return {
12290
+ directory,
12291
+ async seed(indexName) {
12292
+ const log = emptyLog();
12293
+ for (const prefix of [`${indexName}/wal/`, `${indexName}/history/`]) {
12294
+ for (const key of await base.list(prefix)) {
12295
+ const file = loggedFileOf(key);
12296
+ if (file === null || file.indexName !== indexName) continue;
12297
+ setFileBytes(log, key, file.partitionId, await currentLength(key));
12298
+ }
12299
+ }
12300
+ const previous = logs.get(indexName);
12301
+ logs.set(indexName, log);
12302
+ makeCurrent(previous, log, sizes.of(indexName));
12303
+ },
12304
+ forget(indexName) {
12305
+ const log = logs.get(indexName);
12306
+ if (log === void 0) return;
12307
+ logs.delete(indexName);
12308
+ if (log.sizes !== null) {
12309
+ log.sizes.logged = null;
12310
+ refreshStoredSizes(log.sizes);
12311
+ log.sizes = null;
12312
+ }
12313
+ }
12314
+ };
12315
+ }
12316
+
11833
12317
  // src/persistence/durability/manager-state.ts
11834
12318
  function closePartitionWriter(partition) {
11835
12319
  return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
@@ -12244,6 +12728,7 @@ function createIndexUnloading(deps) {
12244
12728
  }
12245
12729
  await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
12246
12730
  deps.forgetSyncState(indexName);
12731
+ deps.dropStoredBytes(indexName);
12247
12732
  },
12248
12733
  async unloadIndex(indexName) {
12249
12734
  const indexState = indexes.get(indexName);
@@ -12254,6 +12739,7 @@ function createIndexUnloading(deps) {
12254
12739
  indexes.delete(indexName);
12255
12740
  metadataWrites.forget(indexName);
12256
12741
  deps.forgetSyncState(indexName);
12742
+ deps.forgetStoredBytes(indexName);
12257
12743
  }
12258
12744
  };
12259
12745
  }
@@ -12721,7 +13207,7 @@ function createSyncStorage(deps) {
12721
13207
 
12722
13208
  // src/persistence/durability/manager-storage.ts
12723
13209
  function createIndexStorageAccess(deps) {
12724
- const { directory, indexes } = deps;
13210
+ const { directory, logBytes, sizes } = deps;
12725
13211
  const archiveAccess = createArchiveCheckpointAccess({
12726
13212
  directory,
12727
13213
  held: deps.heldCheckpointFiles,
@@ -12733,7 +13219,7 @@ function createIndexStorageAccess(deps) {
12733
13219
  directory,
12734
13220
  openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
12735
13221
  inCheckpointTurn: (indexName, work) => {
12736
- const indexState = indexes.get(indexName);
13222
+ const indexState = deps.indexes.get(indexName);
12737
13223
  if (indexState === void 0) return work();
12738
13224
  return checkpointInTurn(indexState, {
12739
13225
  joinsInFlight: false,
@@ -12746,12 +13232,54 @@ function createIndexStorageAccess(deps) {
12746
13232
  const unloading = createIndexUnloading({
12747
13233
  directory,
12748
13234
  hooks: deps.hooks,
12749
- indexes,
13235
+ indexes: deps.indexes,
12750
13236
  metadataWrites: deps.metadataWrites,
12751
13237
  closeWriterReportingFailure: deps.closeWriterReportingFailure,
12752
- forgetSyncState: syncStorage.forget
13238
+ forgetSyncState: syncStorage.forget,
13239
+ forgetStoredBytes: (indexName) => {
13240
+ sizes.forget(indexName);
13241
+ logBytes.forget(indexName);
13242
+ },
13243
+ dropStoredBytes: (indexName) => {
13244
+ sizes.drop(indexName);
13245
+ logBytes.forget(indexName);
13246
+ }
12753
13247
  });
12754
- return { archiveAccess, syncStorage, unloading };
13248
+ async function measureStoredBytes(indexName, saved) {
13249
+ sizes.setSaved(indexName, saved);
13250
+ await logBytes.seed(indexName);
13251
+ }
13252
+ const recoveryDeps = {
13253
+ directory,
13254
+ hooks: deps.hooks,
13255
+ compactionThreshold: deps.compactionThreshold,
13256
+ registerPartition: deps.registerPartition,
13257
+ holdsCheckpointFiles: (indexName) => deps.heldCheckpointFiles.has(indexName),
13258
+ queueMetadataWrite: deps.metadataWrites.queue,
13259
+ syncStorage,
13260
+ measureStoredBytes
13261
+ };
13262
+ const storedBytesAccess = {
13263
+ storedSizesOf: sizes.of,
13264
+ addStoredSizes(indexName, stats) {
13265
+ sizes.of(indexName).addTo(stats);
13266
+ },
13267
+ vectorSavedBytes(indexName) {
13268
+ return sizes.of(indexName).vectorSavedBytes;
13269
+ },
13270
+ async measureRestoredStorage(indexName, restored) {
13271
+ if (restored !== void 0) {
13272
+ await measureStoredBytes(indexName, restored);
13273
+ return;
13274
+ }
13275
+ const decoded = await readSegmentManifest(directoryCheckpointStore(directory), indexName);
13276
+ await measureStoredBytes(
13277
+ indexName,
13278
+ decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED
13279
+ );
13280
+ }
13281
+ };
13282
+ return { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps };
12755
13283
  }
12756
13284
 
12757
13285
  // src/persistence/durability/metadata-write-queue.ts
@@ -13277,7 +13805,7 @@ function createWalWriter(directory, config) {
13277
13805
  }
13278
13806
 
13279
13807
  // src/persistence/durability/partition-registry.ts
13280
- function createPartitionRegistry(directory, segmentMaxBytes) {
13808
+ function createPartitionRegistry(directory, segmentMaxBytes, onIndexHeld) {
13281
13809
  const indexes = /* @__PURE__ */ new Map();
13282
13810
  function indexState(indexName) {
13283
13811
  let state = indexes.get(indexName);
@@ -13291,6 +13819,7 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
13291
13819
  stalledWrites: []
13292
13820
  };
13293
13821
  indexes.set(indexName, state);
13822
+ onIndexHeld(indexName);
13294
13823
  }
13295
13824
  return state;
13296
13825
  }
@@ -13826,7 +14355,7 @@ async function readEarlierSegmentDocuments(store, key) {
13826
14355
  }
13827
14356
 
13828
14357
  // src/persistence/durability/checkpoint-count.ts
13829
- var decoder2 = new TextDecoder();
14358
+ var decoder3 = new TextDecoder();
13830
14359
  async function countPagedPartition(store, partition) {
13831
14360
  const { directory } = store;
13832
14361
  if (directory === void 0 || partition.segments.length === 0) return null;
@@ -13859,7 +14388,7 @@ async function countCurrentLayout(store, manifest) {
13859
14388
  for (const docId of contents.tombstones) live.delete(docId);
13860
14389
  const ids = contents.arrays.idTable;
13861
14390
  for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
13862
- live.add(decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
14391
+ live.add(decoder3.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
13863
14392
  }
13864
14393
  }
13865
14394
  total += live.size;
@@ -14068,19 +14597,18 @@ async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
14068
14597
  const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
14069
14598
  recordWrittenVectors(deps.vectorIndexes, vectors.written);
14070
14599
  await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
14071
- await removeCheckpointGarbage(
14072
- deps.store,
14073
- await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, committed.garbage)
14074
- );
14600
+ const garbage = await checkpointGarbage(deps.store, deps.indexName, committed.manifest);
14601
+ await removeCheckpointGarbage(deps.store, await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, garbage));
14602
+ return committed.saved;
14075
14603
  }
14076
14604
  async function upgradeEarlierLayout(deps, earlier) {
14077
14605
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
14078
14606
  try {
14079
14607
  const loaded = await loadEarlier(deps, earlier);
14080
14608
  const targets = await deps.replay(loaded.checkpoint);
14081
- await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
14609
+ const saved = await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
14082
14610
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
14083
- return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
14611
+ return { checkpoint: targets, documentCount: deps.manager.countDocuments(), saved };
14084
14612
  } catch (error) {
14085
14613
  const failure = error instanceof Error ? error : new Error(String(error));
14086
14614
  deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
@@ -14100,11 +14628,11 @@ async function recoverCheckpoint(deps) {
14100
14628
  const earlier = await earlierLayoutOf(deps, decoded);
14101
14629
  if (earlier !== null) {
14102
14630
  const upgraded = await upgradeEarlierLayout(deps, earlier);
14103
- return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
14631
+ return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount, saved: upgraded.saved };
14104
14632
  }
14105
14633
  if (decoded === null || decoded.kind !== "current") {
14106
14634
  if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
14107
- return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
14635
+ return { checkpoint: await deps.replay([]), upgradedDocumentCount: null, saved: NOTHING_SAVED };
14108
14636
  }
14109
14637
  await deps.store.remove(snapshotBundleKey(deps.indexName));
14110
14638
  if (!deps.keepsUnlistedFiles) {
@@ -14117,7 +14645,11 @@ async function recoverCheckpoint(deps) {
14117
14645
  vectorIndexes: deps.vectorIndexes,
14118
14646
  writeRecords: deps.writeRecords ?? null
14119
14647
  });
14120
- return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
14648
+ return {
14649
+ checkpoint: await deps.replay(checkpoint),
14650
+ upgradedDocumentCount: null,
14651
+ saved: savedBytesOfManifest(decoded.manifest)
14652
+ };
14121
14653
  }
14122
14654
 
14123
14655
  // src/persistence/durability/layout-transition/recovery.ts
@@ -14219,9 +14751,9 @@ async function settleStoredAnalysis(store, metadata, metadataOnly) {
14219
14751
  }
14220
14752
 
14221
14753
  // src/persistence/durability/recover-index.ts
14222
- async function writeCountedMetadata(deps, indexName, documentCount) {
14754
+ async function writeCountedMetadata(deps, indexName, documentCount, savedBytes) {
14223
14755
  await deps.queueMetadataWrite(indexName, async () => {
14224
- const metadata = deps.hooks.buildMetadata(indexName, documentCount);
14756
+ const metadata = deps.hooks.buildMetadata(indexName, documentCount, savedBytes);
14225
14757
  if (metadata === void 0) return;
14226
14758
  const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
14227
14759
  await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
@@ -14316,8 +14848,14 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
14316
14848
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
14317
14849
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
14318
14850
  }
14851
+ await deps.measureStoredBytes(indexName, recovered.saved);
14319
14852
  if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
14320
- await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
14853
+ await writeCountedMetadata(
14854
+ deps,
14855
+ indexName,
14856
+ recovered.upgradedDocumentCount ?? manager.countDocuments(),
14857
+ recovered.saved.bytes
14858
+ );
14321
14859
  }
14322
14860
  return layout.transition === null ? null : removeUnread;
14323
14861
  }
@@ -14333,15 +14871,22 @@ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkp
14333
14871
 
14334
14872
  // src/persistence/durability/manager.ts
14335
14873
  function createDurabilityManager(config, hooks, directoryOverride) {
14336
- const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
14874
+ const sizes = createStoredSizesRecord(true);
14875
+ const logBytes = createLogBytesLedger(createDurableDirectory(config.directory ?? ""), sizes);
14876
+ const { directory } = logBytes;
14337
14877
  const canOffloadCheckpoint = directoryOverride === void 0;
14338
14878
  const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
14339
14879
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
14340
14880
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
14341
14881
  const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
14882
+ const rewriteBytesPerCheckpoint = config.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT;
14342
14883
  const mode = config.mode ?? "sync";
14343
14884
  const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
14344
- const registry = createPartitionRegistry(directory, segmentMaxBytes);
14885
+ const registry = createPartitionRegistry(
14886
+ directory,
14887
+ segmentMaxBytes,
14888
+ (indexName) => hooks.linkStoredSizes(indexName, sizes.hold(indexName))
14889
+ );
14345
14890
  const indexes = registry.indexes;
14346
14891
  const getOrCreateIndexState = registry.indexState;
14347
14892
  const getOrCreatePartition = registry.partition;
@@ -14432,12 +14977,14 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14432
14977
  indexName,
14433
14978
  indexState,
14434
14979
  compactionThreshold,
14980
+ rewriteBytesPerCheckpoint,
14435
14981
  canOffload: canOffloadCheckpoint,
14436
14982
  fromMemory,
14437
14983
  mutationThreshold: checkpointMutationThreshold,
14438
14984
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
14439
14985
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
14440
14986
  keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
14987
+ recordSavedBytes: (saved) => sizes.setSaved(indexName, saved),
14441
14988
  queueMetadataWrite,
14442
14989
  markFatal,
14443
14990
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -14492,28 +15039,22 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14492
15039
  startAsyncFlushTimer();
14493
15040
  return outcomes;
14494
15041
  }
14495
- const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
15042
+ const { archiveAccess, syncStorage, unloading, storedBytesAccess, recoveryDeps } = createIndexStorageAccess({
14496
15043
  directory,
15044
+ logBytes,
15045
+ sizes,
14497
15046
  hooks,
14498
15047
  indexes,
14499
15048
  metadataWrites,
14500
15049
  heldCheckpointFiles,
14501
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
14502
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
14503
- markFatal,
14504
- closeWriterReportingFailure
14505
- });
14506
- const recoveryDeps = {
14507
- directory,
14508
- hooks,
14509
15050
  compactionThreshold,
15051
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
14510
15052
  registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
14511
15053
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
14512
15054
  },
14513
- holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
14514
- queueMetadataWrite,
14515
- syncStorage
14516
- };
15055
+ markFatal,
15056
+ closeWriterReportingFailure
15057
+ });
14517
15058
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
14518
15059
  const recovered = recoverWithLayoutCheckpoint(
14519
15060
  recoveryDeps,
@@ -14610,6 +15151,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14610
15151
  syncStorage: () => syncStorage,
14611
15152
  ...queryHistoryAccess({ kind: "directory", directory }),
14612
15153
  ...archiveAccess,
15154
+ ...storedBytesAccess,
14613
15155
  async shutdown() {
14614
15156
  shuttingDown = true;
14615
15157
  stopTimers();
@@ -14628,6 +15170,31 @@ function createDurabilityManager(config, hooks, directoryOverride) {
14628
15170
  };
14629
15171
  }
14630
15172
 
15173
+ // src/persistence/durability/snapshot-only/index-state.ts
15174
+ function createSnapshotIndexState() {
15175
+ return {
15176
+ mutationsSinceCheckpoint: 0,
15177
+ checkpointInFlight: null,
15178
+ applyChain: Promise.resolve(),
15179
+ unloading: false,
15180
+ appliedSeqNoByPartition: /* @__PURE__ */ new Map()
15181
+ };
15182
+ }
15183
+ function metadataKey(indexName) {
15184
+ return `${indexName}/meta`;
15185
+ }
15186
+ function noteSnapshotCommitted(state, positions, partitionCount) {
15187
+ let recordsLeft = 0;
15188
+ for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
15189
+ if (partitionId >= partitionCount) {
15190
+ state.appliedSeqNoByPartition.delete(partitionId);
15191
+ continue;
15192
+ }
15193
+ recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
15194
+ }
15195
+ state.mutationsSinceCheckpoint = recordsLeft;
15196
+ }
15197
+
14631
15198
  // src/persistence/durability/snapshot-only/checkpoint.ts
14632
15199
  function capture(input, prior, cutoffs) {
14633
15200
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
@@ -14686,10 +15253,21 @@ async function writeCaptured(input, store, prior, captured) {
14686
15253
  });
14687
15254
  const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
14688
15255
  recordWrittenVectors(input.vectorIndexes, vectors.written);
14689
- await removeCheckpointGarbage(
14690
- store,
14691
- await garbageThatNoFieldStillReads(store, input.vectorIndexes, committed.garbage)
14692
- );
15256
+ await input.onCommitted({
15257
+ saved: committed.saved,
15258
+ documentCount: captured.documentCount,
15259
+ partitionCount: captured.partitionCount
15260
+ });
15261
+ const garbage = await checkpointGarbage(store, input.indexName, committed.manifest);
15262
+ await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, input.vectorIndexes, garbage));
15263
+ }
15264
+ async function saveCheckpointMetadata(adapter, hooks, indexName, committed) {
15265
+ const { saved, documentCount, partitionCount } = committed;
15266
+ const checkpointMetadata = hooks.buildMetadata(indexName, documentCount, saved.bytes);
15267
+ if (checkpointMetadata === void 0) return;
15268
+ checkpointMetadata.partitionCount = partitionCount;
15269
+ await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(checkpointMetadata, { checksum: true }));
15270
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount, saved.bytes);
14693
15271
  }
14694
15272
  async function writeSnapshotOnlyCheckpoint(input) {
14695
15273
  const store = adapterCheckpointStore(input.adapter);
@@ -14700,41 +15278,12 @@ async function writeSnapshotOnlyCheckpoint(input) {
14700
15278
  });
14701
15279
  try {
14702
15280
  await writeCaptured(input, store, prior, captured);
14703
- return {
14704
- documentCount: captured.documentCount,
14705
- positions: captured.positions,
14706
- partitionCount: captured.partitionCount
14707
- };
15281
+ return { positions: captured.positions, partitionCount: captured.partitionCount };
14708
15282
  } finally {
14709
15283
  for (const plan of captured.plans.values()) plan.release();
14710
15284
  }
14711
15285
  }
14712
15286
 
14713
- // src/persistence/durability/snapshot-only/index-state.ts
14714
- function createSnapshotIndexState() {
14715
- return {
14716
- mutationsSinceCheckpoint: 0,
14717
- checkpointInFlight: null,
14718
- applyChain: Promise.resolve(),
14719
- unloading: false,
14720
- appliedSeqNoByPartition: /* @__PURE__ */ new Map()
14721
- };
14722
- }
14723
- function metadataKey(indexName) {
14724
- return `${indexName}/meta`;
14725
- }
14726
- function noteSnapshotCommitted(state, positions, partitionCount) {
14727
- let recordsLeft = 0;
14728
- for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
14729
- if (partitionId >= partitionCount) {
14730
- state.appliedSeqNoByPartition.delete(partitionId);
14731
- continue;
14732
- }
14733
- recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
14734
- }
14735
- state.mutationsSinceCheckpoint = recordsLeft;
14736
- }
14737
-
14738
15287
  // src/persistence/durability/snapshot-only/restores.ts
14739
15288
  function toError6(error) {
14740
15289
  return error instanceof Error ? error : new Error(String(error));
@@ -14772,15 +15321,67 @@ function createPendingRestores(onFatalError) {
14772
15321
  };
14773
15322
  }
14774
15323
 
15324
+ // src/persistence/durability/snapshot-only/saved-size.ts
15325
+ function createSnapshotSavedSize(adapter, settleRestore) {
15326
+ const sizes = createStoredSizesRecord(false);
15327
+ const generations = /* @__PURE__ */ new Map();
15328
+ const store = adapterCheckpointStore(adapter);
15329
+ function replace(indexName, saved) {
15330
+ const generation = (generations.get(indexName) ?? 0) + 1;
15331
+ generations.set(indexName, generation);
15332
+ sizes.setSaved(indexName, saved);
15333
+ sizes.setMeasuring(indexName, saved === null);
15334
+ return generation;
15335
+ }
15336
+ async function measureAfterRestore(indexName, generation) {
15337
+ await settleRestore(indexName);
15338
+ const decoded = await readSegmentManifest(store, indexName);
15339
+ if (generations.get(indexName) !== generation) return;
15340
+ replace(indexName, decoded?.kind === "current" ? savedBytesOfManifest(decoded.manifest) : NOTHING_SAVED);
15341
+ }
15342
+ return {
15343
+ record(indexName, saved) {
15344
+ replace(indexName, saved);
15345
+ },
15346
+ hold: sizes.hold,
15347
+ forget(indexName) {
15348
+ generations.delete(indexName);
15349
+ sizes.forget(indexName);
15350
+ },
15351
+ drop(indexName) {
15352
+ generations.delete(indexName);
15353
+ sizes.drop(indexName);
15354
+ },
15355
+ access: {
15356
+ storedSizesOf: sizes.of,
15357
+ addStoredSizes(indexName, stats) {
15358
+ sizes.of(indexName).addTo(stats);
15359
+ },
15360
+ vectorSavedBytes(indexName) {
15361
+ return sizes.of(indexName).vectorSavedBytes;
15362
+ },
15363
+ async measureRestoredStorage(indexName, restored) {
15364
+ if (restored !== void 0) {
15365
+ replace(indexName, restored);
15366
+ return;
15367
+ }
15368
+ const generation = replace(indexName, null);
15369
+ void measureAfterRestore(indexName, generation).catch(() => void 0);
15370
+ }
15371
+ }
15372
+ };
15373
+ }
15374
+
14775
15375
  // src/persistence/durability/snapshot-only/index.ts
14776
15376
  function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14777
15377
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
14778
15378
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
14779
15379
  const indexes = /* @__PURE__ */ new Map();
14780
- const metadataWrites = /* @__PURE__ */ new Map();
15380
+ const metadataWrites = createMetadataWriteQueue();
14781
15381
  const loads = createIndexLoads(hooks.getVectorIndexes);
14782
15382
  const restoreStorage = adapterRestoreStorage(adapter);
14783
15383
  const restores = createPendingRestores(hooks.onFatalError);
15384
+ const savedSize = createSnapshotSavedSize(adapter, settleRestore);
14784
15385
  let checkpointTimer = null;
14785
15386
  let shuttingDown = false;
14786
15387
  function settleRestore(indexName) {
@@ -14791,6 +15392,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14791
15392
  if (state === void 0) {
14792
15393
  state = createSnapshotIndexState();
14793
15394
  indexes.set(indexName, state);
15395
+ hooks.linkStoredSizes(indexName, savedSize.hold(indexName));
14794
15396
  }
14795
15397
  return state;
14796
15398
  }
@@ -14819,23 +15421,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14819
15421
  const manager = hooks.getManager(indexName);
14820
15422
  const metadata = hooks.buildTermsMetadata(indexName);
14821
15423
  if (manager === void 0 || metadata === void 0) return;
14822
- const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
15424
+ const { positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
14823
15425
  adapter,
14824
15426
  indexName,
14825
15427
  metadata,
14826
15428
  manager,
14827
15429
  vectorIndexes: hooks.getVectorIndexes(indexName),
14828
15430
  appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
14829
- rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
14830
- });
14831
- hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
14832
- await queueMetadataWrite(indexName, async () => {
14833
- const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
14834
- if (checkpointMetadata === void 0) return;
14835
- checkpointMetadata.partitionCount = partitionCount;
14836
- const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
14837
- await adapter.save(metadataKey(indexName), metadataBytes);
14838
- hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
15431
+ rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName),
15432
+ onCommitted: async (committed) => {
15433
+ savedSize.record(indexName, committed.saved);
15434
+ hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
15435
+ await queueMetadataWrite(indexName, () => saveCheckpointMetadata(adapter, hooks, indexName, committed));
15436
+ }
14839
15437
  });
14840
15438
  noteSnapshotCommitted(indexState, positions, partitionCount);
14841
15439
  if (publisher !== void 0) {
@@ -14861,16 +15459,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14861
15459
  );
14862
15460
  }
14863
15461
  function queueMetadataWrite(indexName, write) {
14864
- const previous = metadataWrites.get(indexName) ?? Promise.resolve();
14865
- const queued = previous.then(() => settleRestore(indexName)).then(write);
14866
- metadataWrites.set(
14867
- indexName,
14868
- queued.catch(() => void 0)
14869
- );
14870
- return queued;
15462
+ return metadataWrites.queue(indexName, () => settleRestore(indexName).then(write));
14871
15463
  }
14872
- async function saveMetadata(indexName, documentCount) {
14873
- const metadata = hooks.buildMetadata(indexName, documentCount);
15464
+ async function saveMetadata(indexName, documentCount, savedBytes) {
15465
+ const metadata = hooks.buildMetadata(indexName, documentCount, savedBytes);
14874
15466
  if (metadata === void 0) return;
14875
15467
  await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
14876
15468
  }
@@ -14880,9 +15472,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14880
15472
  async function loadIndex(indexName, metadataOnly) {
14881
15473
  await settleRestore(indexName);
14882
15474
  const metaBytes = await adapter.load(metadataKey(indexName));
14883
- if (metaBytes === null) {
14884
- return;
14885
- }
15475
+ if (metaBytes === null) return;
14886
15476
  const { metadata } = await readMetadataEnvelope(metaBytes);
14887
15477
  const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
14888
15478
  let derivedCount;
@@ -14921,6 +15511,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14921
15511
  onLayoutUpgrade: hooks.onLayoutUpgrade
14922
15512
  });
14923
15513
  const indexState = getOrCreateIndexState(indexName);
15514
+ savedSize.record(indexName, recovered.saved);
14924
15515
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
14925
15516
  const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
14926
15517
  if (persistedSeqNo > 0) {
@@ -14929,7 +15520,9 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14929
15520
  }
14930
15521
  if (recovered.upgradedDocumentCount !== null || promoted) {
14931
15522
  const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
14932
- await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
15523
+ await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount, recovered.saved.bytes)).catch(
15524
+ () => void 0
15525
+ );
14933
15526
  }
14934
15527
  }
14935
15528
  async function recordMutation(record) {
@@ -14961,14 +15554,15 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
14961
15554
  try {
14962
15555
  await indexState.applyChain;
14963
15556
  if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
14964
- await metadataWrites.get(indexName);
15557
+ await metadataWrites.pending(indexName);
14965
15558
  } catch (error) {
14966
15559
  indexState.unloading = false;
14967
15560
  throw error;
14968
15561
  }
14969
15562
  }
14970
15563
  indexes.delete(indexName);
14971
- metadataWrites.delete(indexName);
15564
+ metadataWrites.forget(indexName);
15565
+ savedSize.forget(indexName);
14972
15566
  }
14973
15567
  return {
14974
15568
  isActive() {
@@ -15050,6 +15644,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
15050
15644
  }
15051
15645
  },
15052
15646
  ...queryHistoryAccess({ kind: "adapter", adapter }),
15647
+ ...savedSize.access,
15053
15648
  async removeIndex(indexName) {
15054
15649
  await settleRestore(indexName).catch(() => void 0);
15055
15650
  restores.forget(indexName);
@@ -15059,6 +15654,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
15059
15654
  await adapter.delete(key);
15060
15655
  }
15061
15656
  await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
15657
+ savedSize.drop(indexName);
15062
15658
  },
15063
15659
  async unloadIndex(indexName) {
15064
15660
  await settleRestore(indexName);
@@ -15103,7 +15699,7 @@ function vectorFieldsOf(schema, promotion) {
15103
15699
  }
15104
15700
  return Object.keys(fields).length === 0 ? null : fields;
15105
15701
  }
15106
- function metadataFromConfig(indexName, config, partitionCount, documentCount) {
15702
+ function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
15107
15703
  const metadata = {
15108
15704
  indexName,
15109
15705
  schema: config.schema,
@@ -15118,6 +15714,8 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
15118
15714
  engineVersion: VERSION
15119
15715
  };
15120
15716
  metadata.documentCount = documentCount ?? config.documentCount;
15717
+ const recordedSavedBytes = savedBytes ?? config.savedBytes;
15718
+ if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
15121
15719
  const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
15122
15720
  if (vectorFields !== null) {
15123
15721
  metadata.vectorFields = vectorFields;
@@ -15197,6 +15795,7 @@ function metadataConfigOf(entry) {
15197
15795
  ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
15198
15796
  ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
15199
15797
  documentCount: entry.documentCount,
15798
+ savedBytes: entry.savedBytes,
15200
15799
  partitionCount: entry.partitionCount,
15201
15800
  schema: flattenSchema(entry.config.schema),
15202
15801
  language: entry.language.name,
@@ -15275,7 +15874,7 @@ function indexMetadataOf(indexName, entry, partitionCount) {
15275
15874
  }
15276
15875
 
15277
15876
  // src/engine/durability-integration.ts
15278
- function buildMetadata(indexName, hooks, documentCount) {
15877
+ function buildMetadata(indexName, hooks, documentCount, savedBytes) {
15279
15878
  const config = hooks.getIndexConfig(indexName);
15280
15879
  if (config === void 0) {
15281
15880
  return void 0;
@@ -15285,7 +15884,8 @@ function buildMetadata(indexName, hooks, documentCount) {
15285
15884
  indexName,
15286
15885
  config,
15287
15886
  manager === void 0 ? void 0 : metadataPartitionCount(manager),
15288
- documentCount
15887
+ documentCount,
15888
+ savedBytes
15289
15889
  );
15290
15890
  }
15291
15891
  function createDurabilityIntegration(tier, hooks) {
@@ -15293,7 +15893,7 @@ function createDurabilityIntegration(tier, hooks) {
15293
15893
  getManager: hooks.getManager,
15294
15894
  getVectorFieldPaths: hooks.getVectorFieldPaths,
15295
15895
  getVectorIndexes: hooks.getVectorIndexes,
15296
- buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
15896
+ buildMetadata: (indexName, documentCount, savedBytes) => buildMetadata(indexName, hooks, documentCount, savedBytes),
15297
15897
  buildTermsMetadata(indexName) {
15298
15898
  const config = hooks.getTermsConfig(indexName);
15299
15899
  return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
@@ -15308,7 +15908,8 @@ function createDurabilityIntegration(tier, hooks) {
15308
15908
  onCheckpointHeld: hooks.onCheckpointHeld,
15309
15909
  onFatalError: hooks.onFatalError,
15310
15910
  ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
15311
- ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
15911
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved },
15912
+ linkStoredSizes: hooks.linkStoredSizes
15312
15913
  };
15313
15914
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
15314
15915
  async function recordMutation(indexName, docId, operation, document, apply) {
@@ -15429,7 +16030,7 @@ var OLD_REGULAR_FILE = 0;
15429
16030
  var PAX_EXTENDED_HEADER = 120;
15430
16031
  var PAX_HEADER_NAME = "PaxHeader";
15431
16032
  var encoder3 = new TextEncoder();
15432
- var decoder3 = new TextDecoder("utf-8", { fatal: true });
16033
+ var decoder4 = new TextDecoder("utf-8", { fatal: true });
15433
16034
  function writeString(block, offset, length, value) {
15434
16035
  const bytes = encoder3.encode(value);
15435
16036
  block.set(bytes.subarray(0, length), offset);
@@ -15507,7 +16108,7 @@ function readString(block, offset, length) {
15507
16108
  let end = offset;
15508
16109
  while (end < offset + length && block[end] !== 0) end++;
15509
16110
  try {
15510
- return decoder3.decode(block.subarray(offset, end));
16111
+ return decoder4.decode(block.subarray(offset, end));
15511
16112
  } catch {
15512
16113
  invalidArchive("an entry header holds a name that is not UTF-8");
15513
16114
  }
@@ -15544,7 +16145,7 @@ var EQUALS = 61;
15544
16145
  var NEWLINE = 10;
15545
16146
  function decodeRecordText(bytes) {
15546
16147
  try {
15547
- return decoder3.decode(bytes);
16148
+ return decoder4.decode(bytes);
15548
16149
  } catch {
15549
16150
  invalidArchive("a pax extended header is not UTF-8");
15550
16151
  }
@@ -16232,7 +16833,7 @@ function createInvalidationIntegration(deps) {
16232
16833
 
16233
16834
  // src/engine/durability-wiring.ts
16234
16835
  function createDurabilityFromTier(tier, wiring) {
16235
- return createDurabilityIntegration(tier, {
16836
+ const integration = createDurabilityIntegration(tier, {
16236
16837
  checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
16237
16838
  getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
16238
16839
  getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
@@ -16252,8 +16853,25 @@ function createDurabilityFromTier(tier, wiring) {
16252
16853
  },
16253
16854
  queueDiskAdoption: wiring.queueDiskAdoption,
16254
16855
  awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
16255
- createIndexFromMetadata: wiring.createIndexFromMetadata,
16256
- recordCheckpoint: wiring.recordCheckpoint,
16856
+ async createIndexFromMetadata(metadata, loadData) {
16857
+ await wiring.createIndexFromMetadata(metadata, loadData);
16858
+ const entry = wiring.indexRegistry.get(metadata.indexName);
16859
+ if (entry === void 0) return;
16860
+ entry.storedSizes = integration.manager.storedSizesOf(metadata.indexName);
16861
+ entry.storedSizes.recordSavedBytes(entry.savedBytes);
16862
+ },
16863
+ recordCheckpoint(indexName, documentCount, partitionCount, savedBytes) {
16864
+ const entry = wiring.indexRegistry.get(indexName);
16865
+ if (entry === void 0) return;
16866
+ entry.documentCount = documentCount;
16867
+ entry.partitionCount = partitionCount;
16868
+ entry.savedBytes = savedBytes;
16869
+ entry.storedSizes?.recordSavedBytes(savedBytes);
16870
+ },
16871
+ linkStoredSizes(indexName, sizes) {
16872
+ const entry = wiring.indexRegistry.get(indexName);
16873
+ if (entry !== void 0) entry.storedSizes = sizes;
16874
+ },
16257
16875
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
16258
16876
  onCheckpointHeld: wiring.emitCheckpointHeld,
16259
16877
  onFatalError: wiring.emitFatalError,
@@ -16267,6 +16885,7 @@ function createDurabilityFromTier(tier, wiring) {
16267
16885
  }
16268
16886
  }
16269
16887
  });
16888
+ return integration;
16270
16889
  }
16271
16890
  async function reloadIndexAndCopies(orchestrator, durability, indexName) {
16272
16891
  const manager = durability?.manager;
@@ -16612,7 +17231,7 @@ function joined2(parts, total) {
16612
17231
  return bytes;
16613
17232
  }
16614
17233
  async function* exportLines(source, maxLineBytes) {
16615
- const decoder5 = new TextDecoder("utf-8", { fatal: true });
17234
+ const decoder6 = new TextDecoder("utf-8", { fatal: true });
16616
17235
  let parts = [];
16617
17236
  let pending = 0;
16618
17237
  let overflowing = false;
@@ -16629,7 +17248,7 @@ async function* exportLines(source, maxLineBytes) {
16629
17248
  parts = [];
16630
17249
  pending = 0;
16631
17250
  try {
16632
- return { line, text: decoder5.decode(bytes) };
17251
+ return { line, text: decoder6.decode(bytes) };
16633
17252
  } catch {
16634
17253
  return { line, refused: "not-utf8" };
16635
17254
  }
@@ -17164,10 +17783,10 @@ async function readSegment(checkpoint, ref) {
17164
17783
  });
17165
17784
  return decodeKeywordSegment(payload, false);
17166
17785
  }
17167
- var decoder4 = new TextDecoder();
17786
+ var decoder5 = new TextDecoder();
17168
17787
  function idAt(contents, ordinal) {
17169
17788
  const ids = contents.arrays.idTable;
17170
- return decoder4.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
17789
+ return decoder5.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
17171
17790
  }
17172
17791
  async function winnersOf(checkpoint, partition) {
17173
17792
  const winners = /* @__PURE__ */ new Map();
@@ -17688,8 +18307,9 @@ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
17688
18307
  partitionCount ?? void 0
17689
18308
  );
17690
18309
  }
17691
- function restoredRegistryEntry(prepared, manager, deps) {
18310
+ function restoredRegistryEntry(indexName, prepared, manager, deps) {
17692
18311
  return {
18312
+ storedSizes: deps.durability?.manager.storedSizesOf(indexName),
17693
18313
  config: prepared.config,
17694
18314
  language: prepared.language,
17695
18315
  embeddingAdapter: prepared.embeddingAdapter,
@@ -17789,13 +18409,17 @@ async function commitToStorage(indexName, restored, previous, deps) {
17789
18409
  for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
17790
18410
  await staging.commit();
17791
18411
  } catch (error) {
17792
- if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
18412
+ if (previousPartitions > 0) {
18413
+ durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
18414
+ await durable.measureRestoredStorage?.(indexName).catch(() => void 0);
18415
+ }
17793
18416
  throw error;
17794
18417
  }
17795
18418
  for (const paged of pagedSegmentsOf(previous)) paged.close();
17796
18419
  const restoredSegments = pagedSegmentsOf(restored.detached.manager);
17797
18420
  for (const paged of restoredSegments) paged.suspend();
17798
18421
  await staging.finish();
18422
+ await durable.measureRestoredStorage?.(indexName, restored.saved ?? void 0).catch(() => void 0);
17799
18423
  const relocate = await staging.relocation().catch(() => null);
17800
18424
  for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
17801
18425
  if (relocate === null) return;
@@ -17804,6 +18428,7 @@ async function commitToStorage(indexName, restored, previous, deps) {
17804
18428
  function activate(indexName, restored, entry, deps) {
17805
18429
  deps.executor.installDetachedIndex(restored.detached);
17806
18430
  deps.indexRegistry.set(indexName, entry);
18431
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
17807
18432
  restored.detached.attach();
17808
18433
  deps.clearAnalysisStale(indexName);
17809
18434
  const { prepared } = restored;
@@ -17843,6 +18468,7 @@ async function createFresh(indexName, restored, entry, deps) {
17843
18468
  deps.copies.desyncIndex(indexName);
17844
18469
  deps.executor.installDetachedIndex(restored.detached);
17845
18470
  deps.indexRegistry.set(indexName, entry);
18471
+ entry.storedSizes?.recordSavedBytes(entry.savedBytes);
17846
18472
  try {
17847
18473
  await deps.indexState.registerReplacing(indexName, async () => {
17848
18474
  await commitToStorage(indexName, restored, void 0, deps);
@@ -17861,9 +18487,10 @@ async function stageMetadata(indexName, staging, entry) {
17861
18487
  await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
17862
18488
  }
17863
18489
  async function installRestoredIndex(indexName, restored, deps, history = null) {
17864
- const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
18490
+ const entry = restoredRegistryEntry(indexName, restored.prepared, restored.detached.manager, deps);
17865
18491
  const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
17866
18492
  if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
18493
+ if (restored.saved !== null) entry.savedBytes = restored.saved.bytes;
17867
18494
  let replaced;
17868
18495
  try {
17869
18496
  if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
@@ -17996,7 +18623,7 @@ async function loadStagedIndex(indexName, staged, staging, store, deps) {
17996
18623
  deps.executor.discardDetachedIndex(detached);
17997
18624
  throw error;
17998
18625
  }
17999
- return { prepared, detached, staging, checkpoint: manifest.checkpoint };
18626
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint, saved: savedBytesOfManifest(manifest) };
18000
18627
  }
18001
18628
  async function restoreArchive(indexName, reader, deps) {
18002
18629
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
@@ -18097,7 +18724,7 @@ function insertSnapshotDocuments(envelope, manager) {
18097
18724
  for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
18098
18725
  }
18099
18726
  async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
18100
- const entry = restoredRegistryEntry(prepared, built.manager, deps);
18727
+ const entry = restoredRegistryEntry(indexName, prepared, built.manager, deps);
18101
18728
  const archive = await buildArchiveFromMemory({
18102
18729
  indexName,
18103
18730
  metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
@@ -18121,7 +18748,7 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
18121
18748
  let manifest;
18122
18749
  try {
18123
18750
  insertSnapshotDocuments(envelope, built.manager);
18124
- if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
18751
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [], saved: null };
18125
18752
  manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
18126
18753
  } catch (err) {
18127
18754
  deps.executor.discardDetachedIndex(built);