@delali/narsil 0.3.1-next.180 → 0.3.1-next.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
  2. package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
  3. package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
  4. package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
  5. package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
  6. package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
  7. package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
  8. package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
  9. package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
  10. package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
  11. package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
  12. package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
  13. package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
  14. package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
  15. package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
  16. package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
  17. package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
  18. package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
  19. package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
  20. package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
  21. package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
  22. package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
  23. package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
  24. package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
  25. package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
  26. package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
  27. package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
  28. package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
  29. package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
  30. package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
  31. package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
  32. package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
  33. package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
  34. package/dist/client.d.ts +6 -6
  35. package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
  36. package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
  37. package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
  38. package/dist/distribution.d.ts +4 -4
  39. package/dist/distribution.mjs +391 -138
  40. package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
  41. package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
  42. package/dist/index.browser.mjs +1 -1
  43. package/dist/index.d.ts +4 -4
  44. package/dist/index.mjs +1 -1
  45. package/dist/index.react-native.mjs +1 -1
  46. package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
  47. package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
  48. package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
  49. package/dist/partial-words.browser.mjs +1 -1
  50. package/dist/partial-words.mjs +1 -1
  51. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
  52. package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
  53. package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
  54. package/dist/query-suggestions/merge-worker.mjs +4 -4
  55. package/dist/query-suggestions.browser.mjs +1 -1
  56. package/dist/query-suggestions.d.ts +1 -1
  57. package/dist/query-suggestions.mjs +2 -2
  58. package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
  59. package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
  60. package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
  61. package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
  62. package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
  63. package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
  64. package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
  65. package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
  66. package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
  67. package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
  68. package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
  69. package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
  70. package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
  71. package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
  72. package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
  73. package/dist/react-native/partial-words.mjs +1 -1
  74. package/dist/react-native/query-suggestions.mjs +1 -1
  75. package/dist/react-native/sync.mjs +1 -1
  76. package/dist/react-native/synonyms.mjs +1 -1
  77. package/dist/react.d.ts +7 -7
  78. package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
  79. package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
  80. package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
  81. package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
  82. package/dist/server.d.ts +8 -8
  83. package/dist/server.mjs +16 -16
  84. package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
  85. package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
  86. package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
  87. package/dist/sync/filesystem.d.ts +2 -2
  88. package/dist/sync/filesystem.mjs +6 -6
  89. package/dist/sync/indexeddb.d.ts +2 -2
  90. package/dist/sync.browser.mjs +1 -1
  91. package/dist/sync.d.ts +3 -3
  92. package/dist/sync.mjs +1 -1
  93. package/dist/synonyms.browser.mjs +1 -1
  94. package/dist/synonyms.mjs +1 -1
  95. package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
  96. package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
  97. package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
  98. package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
  99. package/dist/workers/entry.mjs +7 -7
  100. package/openapi.json +32 -0
  101. package/package.json +7 -7
@@ -1,7 +1,7 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
2
- import { C as ClusterNamespace } from './config-CIpLXXxm.js';
2
+ import { C as ClusterNamespace } from './config-D9WR3MhO.js';
3
3
  import { l as RecordedQuery } from './query-suggestions-CBYkrQ5T.js';
4
- import { E as ErrorDetail, j as TaskStore } from './tasks-CXtQ3Zpp.js';
4
+ import { E as ErrorDetail, j as TaskStore } from './tasks-X2IJQc8J.js';
5
5
 
6
6
  /**
7
7
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -1,4 +1,4 @@
1
- import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-2C3BXZGA.mjs';
1
+ import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-CWXR7OP4.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
3
3
 
4
4
  // src/core/partition/frozen/share.ts
@@ -1,12 +1,12 @@
1
- import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, FILE_ASSEMBLY_CHUNK_BYTES, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL, resumePagedFile, suspendPagedFile, cachedFileBytes, requireNativePage, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, CACHED_DOCUMENT_BLOCK, failNativePage, readFileRangeLater, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-ZB5X5WAK.mjs';
1
+ import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL, resumePagedFile, suspendPagedFile, cachedFileBytes, requireNativePage, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, CACHED_DOCUMENT_BLOCK, failNativePage, readFileRangeLater, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-TTTBE4TB.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
3
3
  import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
4
- import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-KHJPMZE2.mjs';
5
- import { DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-QDZJM7NI.mjs';
4
+ import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-PK3BGGHO.mjs';
5
+ import { DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-MSIN4NM3.mjs';
6
6
  import { encode } from './chunk-CV3Q2EKO.mjs';
7
7
  import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
8
- import { readHeader, HEADER_SIZE, crc32Init, crc32Update, crc32Final, requireWithinFileLimit, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
9
- import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-2C3BXZGA.mjs';
8
+ import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
9
+ import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-CWXR7OP4.mjs';
10
10
  import { encodeDocument } from './chunk-KLVMK3R5.mjs';
11
11
  import { decode } from './chunk-GXCDWLQI.mjs';
12
12
  import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-X5NHWOXT.mjs';
@@ -3575,6 +3575,9 @@ async function mergePagedWindow(input) {
3575
3575
  for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
3576
3576
  }
3577
3577
  }
3578
+ for (const docId of input.replacedAfterWindow ?? []) {
3579
+ for (const source of sources) source.segment.tombstoneDocument(docId);
3580
+ }
3578
3581
  const tombstones = /* @__PURE__ */ new Set();
3579
3582
  if (!input.dropsEveryTombstone) {
3580
3583
  for (const source of sources) {
@@ -3605,6 +3608,61 @@ async function mergePagedWindow(input) {
3605
3608
  }
3606
3609
  }
3607
3610
 
3611
+ // src/persistence/paged/segment-scan.ts
3612
+ async function openPagedSegmentScan(directory, ref) {
3613
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
3614
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
3615
+ const segment = createPagedFrozenSegment(snapshot, cache);
3616
+ const { paged } = segment;
3617
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
3618
+ const ids = segment.arrays.idTable;
3619
+ const tombstones = new Set(snapshot.tombstones);
3620
+ return {
3621
+ count: snapshot.documentCount,
3622
+ schema: snapshot.schema,
3623
+ tombstones,
3624
+ idAt: (ordinal) => ids.idAt(ordinal),
3625
+ holds: (docId) => ids.ordinalOf(docId) >= 0,
3626
+ storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
3627
+ close: () => {
3628
+ paged.close();
3629
+ cache.clear();
3630
+ }
3631
+ };
3632
+ }
3633
+ async function checkPagedSegmentFully(directory, ref) {
3634
+ const listing = await pagedListingOf(directory, ref);
3635
+ await requirePagedPayloadChecksum(directory, listing);
3636
+ const snapshot = openPagedSnapshot(listing);
3637
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
3638
+ const segment = createPagedFrozenSegment(snapshot, cache);
3639
+ const { paged } = segment;
3640
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
3641
+ try {
3642
+ await checkPagedContents(segment, cache);
3643
+ return snapshot.schema;
3644
+ } finally {
3645
+ paged.close();
3646
+ cache.clear();
3647
+ }
3648
+ }
3649
+ async function bytesThatSegmentsHoldInMemory(directory, manifest) {
3650
+ let bytes = 0;
3651
+ for (const partition of manifest.partitions) {
3652
+ for (const ref of partition.segments) {
3653
+ const paged = await isPagedSegmentFile(directory, ref.key);
3654
+ bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
3655
+ }
3656
+ }
3657
+ return bytes;
3658
+ }
3659
+ function countsAsLatestHolder(scans, at, docId) {
3660
+ for (let later = at + 1; later < scans.length; later++) {
3661
+ if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
3662
+ }
3663
+ return true;
3664
+ }
3665
+
3608
3666
  // src/persistence/durability/partition-checkpoint.ts
3609
3667
  function checkpointLastSeqNo(checkpoint, partitionId) {
3610
3668
  for (const c of checkpoint) {
@@ -3623,6 +3681,25 @@ function checkpointPrimaryTerm(checkpoint, partitionId) {
3623
3681
  return 0;
3624
3682
  }
3625
3683
 
3684
+ // src/persistence/durability/stored-bytes.ts
3685
+ var NOTHING_SAVED = { bytes: 0, vectorBytes: 0, partitions: [] };
3686
+ function savedBytesOfManifest(manifest) {
3687
+ let bytes = 0;
3688
+ const partitions = [];
3689
+ for (const partition of manifest.partitions) {
3690
+ let partitionBytes = 0;
3691
+ for (const segment of partition.segments) partitionBytes += segment.bytes;
3692
+ partitions.push({ partitionId: partition.partitionId, bytes: partitionBytes });
3693
+ bytes += partitionBytes;
3694
+ }
3695
+ let vectorBytes = 0;
3696
+ for (const field of manifest.vectors) {
3697
+ for (const file of field.files) vectorBytes += file.bytes;
3698
+ for (const part of field.graphParts) vectorBytes += part.bytes;
3699
+ }
3700
+ return { bytes: bytes + vectorBytes, vectorBytes, partitions };
3701
+ }
3702
+
3626
3703
  // src/persistence/durability/commit-marker.ts
3627
3704
  var COMMIT_MARKER_SLOT_SIZE = 36;
3628
3705
  var COMMIT_MARKER_SLOT_COUNT = 2;
@@ -4457,49 +4534,34 @@ function retainedTombstones(frozen, contents) {
4457
4534
  }
4458
4535
  return [...retained];
4459
4536
  }
4460
- async function compactPartitionSegments(input) {
4461
- const { segments, compactionThreshold } = input;
4462
- if (segments.length <= compactionThreshold) {
4463
- return { segments, nextSegmentId: input.nextSegmentId };
4464
- }
4465
- const paged = input.paged === true;
4466
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
4467
- if (window === null) {
4468
- return { segments, nextSegmentId: input.nextSegmentId };
4469
- }
4470
- const windowRefs = segments.slice(window.start, window.start + window.windowLength);
4471
- const replacing = (written2) => ({
4472
- segments: [
4473
- ...segments.slice(0, window.start),
4474
- ...written2.segments,
4475
- ...segments.slice(window.start + window.windowLength)
4476
- ],
4477
- nextSegmentId: written2.nextSegmentId
4478
- });
4479
- if (paged) {
4537
+ async function mergeSegmentWindow(input, window) {
4538
+ if (input.paged === true) {
4480
4539
  const { directory } = input.store;
4481
- if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
4482
- return replacing(
4483
- await mergePagedWindow({
4484
- store: input.store,
4485
- directory,
4486
- indexName: input.indexName,
4487
- partitionId: input.partitionId,
4488
- schema: input.schema,
4489
- partialWordAnalysis: input.partialWordAnalysis,
4490
- textAnalysis: input.textAnalysis,
4491
- nextSegmentId: input.nextSegmentId,
4492
- refs: windowRefs,
4493
- dropsEveryTombstone: window.start === 0
4494
- })
4495
- );
4540
+ if (directory === void 0) return null;
4541
+ return mergePagedWindow({
4542
+ store: input.store,
4543
+ directory,
4544
+ indexName: input.indexName,
4545
+ partitionId: input.partitionId,
4546
+ schema: input.schema,
4547
+ partialWordAnalysis: input.partialWordAnalysis,
4548
+ textAnalysis: input.textAnalysis,
4549
+ nextSegmentId: input.nextSegmentId,
4550
+ refs: window.refs,
4551
+ dropsEveryTombstone: window.startsTheList,
4552
+ ...window.replacedAfterWindow === void 0 ? {} : { replacedAfterWindow: window.replacedAfterWindow }
4553
+ });
4496
4554
  }
4497
4555
  const contents = [];
4498
- for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
4556
+ for (const ref of window.refs) contents.push(await readSegmentFile(input.store, ref));
4499
4557
  const frozen = contents.map(frozenOf);
4500
4558
  applyTimeOrder(frozen, contents);
4559
+ for (const docId of window.replacedAfterWindow ?? []) {
4560
+ for (const segment of frozen) segment.tombstoneDocument(docId);
4561
+ }
4501
4562
  const writeRecords = writeRecordsOf2(frozen, contents);
4502
- const written = await writeSegmentsOf({
4563
+ const keepsTombstones = !(window.startsTheList && window.replacedAfterWindow !== void 0);
4564
+ return writeSegmentsOf({
4503
4565
  store: input.store,
4504
4566
  indexName: input.indexName,
4505
4567
  partitionId: input.partitionId,
@@ -4508,11 +4570,133 @@ async function compactPartitionSegments(input) {
4508
4570
  partialWordAnalysis: input.partialWordAnalysis,
4509
4571
  textAnalysis: input.textAnalysis,
4510
4572
  selections: frozen.map((segment) => ({ segment })),
4511
- tombstones: retainedTombstones(frozen, contents),
4573
+ tombstones: keepsTombstones ? retainedTombstones(frozen, contents) : [],
4512
4574
  mergesIntoOneFile: true,
4513
4575
  ...writeRecords === void 0 ? {} : { writeRecords }
4514
4576
  });
4515
- return replacing(written);
4577
+ }
4578
+ async function compactPartitionSegments(input) {
4579
+ const { segments, compactionThreshold } = input;
4580
+ if (segments.length <= compactionThreshold) {
4581
+ return { segments, nextSegmentId: input.nextSegmentId };
4582
+ }
4583
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, input.paged === true);
4584
+ if (window === null) {
4585
+ return { segments, nextSegmentId: input.nextSegmentId };
4586
+ }
4587
+ const written = await mergeSegmentWindow(input, {
4588
+ refs: segments.slice(window.start, window.start + window.windowLength),
4589
+ startsTheList: window.start === 0
4590
+ });
4591
+ if (written === null) return { segments, nextSegmentId: input.nextSegmentId };
4592
+ return {
4593
+ segments: [
4594
+ ...segments.slice(0, window.start),
4595
+ ...written.segments,
4596
+ ...segments.slice(window.start + window.windowLength)
4597
+ ],
4598
+ nextSegmentId: written.nextSegmentId
4599
+ };
4600
+ }
4601
+
4602
+ // src/persistence/durability/segment/removed-documents.ts
4603
+ var decoder2 = new TextDecoder();
4604
+ async function readSegmentIds(store, ref) {
4605
+ const { directory } = store;
4606
+ if (directory !== void 0 && await isPagedSegmentFile(directory, ref.key)) {
4607
+ const scan = await openPagedSegmentScan(directory, ref);
4608
+ try {
4609
+ const ids2 = new Array(scan.count);
4610
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) ids2[ordinal] = scan.idAt(ordinal);
4611
+ return { ids: ids2, tombstones: [...scan.tombstones] };
4612
+ } finally {
4613
+ scan.close();
4614
+ }
4615
+ }
4616
+ const contents = await readSegmentFile(store, ref);
4617
+ const table = contents.arrays.idTable;
4618
+ const ids = new Array(contents.arrays.documentCount);
4619
+ for (let ordinal = 0; ordinal < ids.length; ordinal++) {
4620
+ ids[ordinal] = decoder2.decode(table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]));
4621
+ }
4622
+ return { ids, tombstones: contents.tombstones };
4623
+ }
4624
+ async function documentsReplacedLater(store, segments) {
4625
+ const later = /* @__PURE__ */ new Set();
4626
+ const replaced = new Array(segments.length);
4627
+ for (let at = segments.length - 1; at >= 0; at--) {
4628
+ const { ids, tombstones } = await readSegmentIds(store, segments[at]);
4629
+ const found = [];
4630
+ for (const docId of ids) if (later.has(docId)) found.push(docId);
4631
+ replaced[at] = found;
4632
+ if (at === 0) break;
4633
+ for (const docId of ids) later.add(docId);
4634
+ for (const docId of tombstones) later.add(docId);
4635
+ }
4636
+ return replaced;
4637
+ }
4638
+ function exceedsRemovedShare(removed, held) {
4639
+ return held > 0 && removed > held * REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE;
4640
+ }
4641
+ function freedBytesOf(ref, replacedCount) {
4642
+ return ref.docCount === 0 ? 0 : ref.bytes * replacedCount / ref.docCount;
4643
+ }
4644
+ function chooseRewrites(segments, replaced, budget) {
4645
+ let held = 0;
4646
+ let removed = 0;
4647
+ for (let at = 0; at < segments.length; at++) {
4648
+ held += segments[at].docCount;
4649
+ removed += replaced[at].length;
4650
+ }
4651
+ const candidates = [];
4652
+ for (let at = 0; at < segments.length; at++) if (replaced[at].length > 0) candidates.push(at);
4653
+ candidates.sort(
4654
+ (left, right) => freedBytesOf(segments[right], replaced[right].length) - freedBytesOf(segments[left], replaced[left].length)
4655
+ );
4656
+ const chosen = [];
4657
+ for (const at of candidates) {
4658
+ if (!exceedsRemovedShare(removed, held)) break;
4659
+ const bytes = segments[at].bytes;
4660
+ if (chosen.length > 0 && budget.spentBytes + bytes > budget.limitBytes) continue;
4661
+ chosen.push(at);
4662
+ budget.spentBytes += bytes;
4663
+ held -= replaced[at].length;
4664
+ removed -= replaced[at].length;
4665
+ }
4666
+ return chosen.sort((left, right) => left - right);
4667
+ }
4668
+ function holdsSomething(ref) {
4669
+ return ref.docCount > 0 || ref.tombstoneCount > 0;
4670
+ }
4671
+ async function rewriteRemovedDocuments(input) {
4672
+ const { segments, liveDocumentCount } = input;
4673
+ const unchanged = { segments, nextSegmentId: input.nextSegmentId };
4674
+ if (liveDocumentCount === void 0) return unchanged;
4675
+ let held = 0;
4676
+ for (const ref of segments) held += ref.docCount;
4677
+ if (!exceedsRemovedShare(held - liveDocumentCount, held)) return unchanged;
4678
+ const replaced = await documentsReplacedLater(input.store, segments);
4679
+ const chosen = new Set(chooseRewrites(segments, replaced, input.budget));
4680
+ if (chosen.size === 0) return unchanged;
4681
+ const rewritten = [];
4682
+ let nextSegmentId = input.nextSegmentId;
4683
+ for (let at = 0; at < segments.length; at++) {
4684
+ if (!chosen.has(at)) {
4685
+ rewritten.push(segments[at]);
4686
+ continue;
4687
+ }
4688
+ const written = await mergeSegmentWindow(
4689
+ { ...input, nextSegmentId },
4690
+ { refs: [segments[at]], startsTheList: rewritten.length === 0, replacedAfterWindow: replaced[at] }
4691
+ );
4692
+ if (written === null) {
4693
+ rewritten.push(segments[at]);
4694
+ continue;
4695
+ }
4696
+ nextSegmentId = written.nextSegmentId;
4697
+ rewritten.push(...written.segments.filter(holdsSomething));
4698
+ }
4699
+ return { segments: rewritten, nextSegmentId };
4516
4700
  }
4517
4701
 
4518
4702
  // src/persistence/durability/segment/write.ts
@@ -4637,8 +4821,21 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
4637
4821
  compactionThreshold: context.compactionThreshold,
4638
4822
  paged: context.input.metadata.textStorage === "disk"
4639
4823
  });
4640
- segments = compacted.segments;
4641
- nextSegmentId = compacted.nextSegmentId;
4824
+ const rewritten = await rewriteRemovedDocuments({
4825
+ store: context.input.store,
4826
+ indexName: context.indexName,
4827
+ partitionId,
4828
+ schema: context.input.metadata.schema,
4829
+ partialWordAnalysis: context.partialWordAnalysis,
4830
+ textAnalysis: context.textAnalysis,
4831
+ segments: compacted.segments,
4832
+ nextSegmentId: compacted.nextSegmentId,
4833
+ paged: context.input.metadata.textStorage === "disk",
4834
+ liveDocumentCount: context.input.liveDocuments?.find((entry) => entry.partitionId === partitionId)?.documentCount,
4835
+ budget: context.rewriteBudget
4836
+ });
4837
+ segments = rewritten.segments;
4838
+ nextSegmentId = rewritten.nextSegmentId;
4642
4839
  return { partitionId, nextSegmentId, segments };
4643
4840
  }
4644
4841
  function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
@@ -4663,7 +4860,11 @@ async function writeCheckpointSegments(input) {
4663
4860
  partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
4664
4861
  textAnalysis,
4665
4862
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
4666
- compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
4863
+ compactionThreshold: resolveCompactionThreshold(input.compactionThreshold),
4864
+ rewriteBudget: {
4865
+ spentBytes: 0,
4866
+ limitBytes: input.rewriteBytesPerCheckpoint ?? DEFAULT_REWRITE_BYTES_PER_CHECKPOINT
4867
+ }
4667
4868
  };
4668
4869
  const checkpointByPartition = /* @__PURE__ */ new Map();
4669
4870
  for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
@@ -4710,9 +4911,9 @@ async function commitCheckpointManifest(store, metadata, written, vectors) {
4710
4911
  };
4711
4912
  const packed = await encodeSegmentManifest(manifest);
4712
4913
  await store.write(manifestKey(indexName), packed.chunks);
4713
- return { garbage: await unreferencedKeys(store, indexName, manifest) };
4914
+ return { manifest, saved: savedBytesOfManifest(manifest) };
4714
4915
  }
4715
- async function unreferencedKeys(store, indexName, manifest) {
4916
+ async function checkpointGarbage(store, indexName, manifest) {
4716
4917
  const referenced = manifestReferencedKeys(manifest);
4717
4918
  const stored = await store.list(segmentsPrefix(indexName));
4718
4919
  return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
@@ -4723,4 +4924,4 @@ async function removeCheckpointGarbage(store, garbage) {
4723
4924
  }
4724
4925
  }
4725
4926
 
4726
- export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry, bytesOfResidentSections, captureWholePartition, checkPagedContents, checkPagedSelections, checkSegmentHeader, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, requirePagedPayloadChecksum, residentSectionsOf, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
4927
+ export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, residentSectionsOf, savedBytesOfManifest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
@@ -1,4 +1,4 @@
1
- import { withOpenIndex, oneAtATime, validateBM25Params, storedBM25Params, emitSyncChange } from './chunk-2C3BXZGA.mjs';
1
+ import { withOpenIndex, oneAtATime, validateBM25Params, storedBM25Params, emitSyncChange } from './chunk-CWXR7OP4.mjs';
2
2
  import { ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
3
3
 
4
4
  // src/engine/bm25-update.ts
@@ -1,5 +1,5 @@
1
1
  import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
2
- import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-QDZJM7NI.mjs';
2
+ import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-MSIN4NM3.mjs';
3
3
  import { nodeThreadId } from './chunk-DG6WIEB3.mjs';
4
4
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
5
5
  import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
@@ -1,4 +1,4 @@
1
- import { bindResponseStop, sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-XTSFRWAS.mjs';
1
+ import { bindResponseStop, sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-FIXUVFUC.mjs';
2
2
  import { ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
3
3
  import { rm } from 'fs/promises';
4
4
  import path from 'path';
@@ -1,4 +1,4 @@
1
- import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-2C3BXZGA.mjs';
1
+ import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-CWXR7OP4.mjs';
2
2
  import { flattenSchema } from './chunk-X5NHWOXT.mjs';
3
3
  import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-4PY4JYMY.mjs';
4
4
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
@@ -1,7 +1,7 @@
1
- import { loadFailed, cachedFileBytes, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile, COLD_GRAPH_PART_VECTORS, decodeSections, readListedFile, requireListedChecksum, requireListedSize, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, failNativePage, COLD_WALK_NODES_READ_AHEAD } from './chunk-ZB5X5WAK.mjs';
1
+ import { loadFailed, cachedFileBytes, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile, COLD_GRAPH_PART_VECTORS, decodeSections, readListedFile, requireListedChecksum, requireListedSize, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, failNativePage, COLD_WALK_NODES_READ_AHEAD } from './chunk-TTTBE4TB.mjs';
2
2
  import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, requireBoundedUpperLists, osqRecordBytes, osqBitsOf, countDead, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, createFixedBuffer } from './chunk-TNEU32EN.mjs';
3
- import { HEADER_SIZE, openChecksummedEnvelope, crc32, readHeader } from './chunk-JCWC75NH.mjs';
4
- import { yieldToEventLoop, compareScores } from './chunk-2C3BXZGA.mjs';
3
+ import { HEADER_SIZE, openChecksummedEnvelope, crc32, readHeader } from './chunk-C77OBNPP.mjs';
4
+ import { yieldToEventLoop, compareScores } from './chunk-CWXR7OP4.mjs';
5
5
  import { decode, decodeMulti } from './chunk-GXCDWLQI.mjs';
6
6
  import { signalOfSearch, requireSearchActive } from './chunk-DG6WIEB3.mjs';
7
7
  import { MIN_M, MAX_M, MAX_LAYER_CAP } from './chunk-ZXX5GZEX.mjs';
@@ -12680,7 +12680,7 @@ function resolvePartitionInsertOptions(config, analysis, options) {
12680
12680
 
12681
12681
  // src/core/segment-codec-code.ts
12682
12682
  var segmentCodec = codeLoader(async () => {
12683
- const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-24Z3MMQI.mjs')]);
12683
+ const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./segment-codec-leaf-ACALKS2C.mjs')]);
12684
12684
  code7.bindSegmentCodecHost(optionalCodeHost(), msgpack);
12685
12685
  return code7;
12686
12686
  }, "segment codec");
@@ -12688,7 +12688,7 @@ var loadSegmentCodec = segmentCodec.load;
12688
12688
 
12689
12689
  // src/query-suggestions/store-code.ts
12690
12690
  var suggestionStore = codeLoader(async () => {
12691
- const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-ESVLL4JO.mjs')]);
12691
+ const [msgpack, code7] = await Promise.all([loadMessagePack(), import('./store-leaf-CWHHEDRK.mjs')]);
12692
12692
  code7.bindSuggestionStoreHost(optionalCodeHost(), msgpack);
12693
12693
  return code7;
12694
12694
  }, "query suggestions store");
@@ -12769,7 +12769,7 @@ var storageCode = codeLoader(async () => {
12769
12769
  loadSegmentMerge(),
12770
12770
  loadSuggestionStore(),
12771
12771
  loadVectorCore(),
12772
- import('./storage-leaf-KIA7XO6Y.mjs')
12772
+ import('./storage-leaf-OJ55R4C5.mjs')
12773
12773
  ]);
12774
12774
  code7.bindStorageHost(optionalCodeHost(), digest2, msgpack, segmentCodec2, segmentMerge2, suggestionStore2, vectorCore2);
12775
12775
  return code7;
@@ -19104,8 +19104,7 @@ function assembleFrozenSegment(segmentId, source2, documentSource, sharedSnapsho
19104
19104
  if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
19105
19105
  return removals.spellingsOf(ordinal);
19106
19106
  },
19107
- prepareTombstone(docId, removedSpellings, analysis) {
19108
- const ordinal = docStore.getInternalId(docId);
19107
+ prepareTombstone(docId, removedSpellings, analysis, ordinal = docStore.getInternalId(docId)) {
19109
19108
  if (ordinal === void 0 || tombstones.has(ordinal)) return void 0;
19110
19109
  const publishTombstone = tombstones.prepare(ordinal);
19111
19110
  const strings4 = analysis === void 0 ? null : prepareFrozenStrings(
@@ -21493,7 +21492,7 @@ function hasLanguage(name) {
21493
21492
 
21494
21493
  // src/narsil/restore-queue-code.ts
21495
21494
  var restoreQueue = codeLoader(async () => {
21496
- const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-PUVBN6L3.mjs')]);
21495
+ const [suggestionStore2, code7] = await Promise.all([loadSuggestionStore(), import('./restore-queue-LF4DTVJC.mjs')]);
21497
21496
  code7.bindRestoreQueueHost(optionalCodeHost(), suggestionStore2);
21498
21497
  return code7;
21499
21498
  }, "restore");
@@ -22068,7 +22067,7 @@ var loadEmbedCode = embedCode.load;
22068
22067
 
22069
22068
  // src/engine/optional-writes-code.ts
22070
22069
  var optionalWrites = codeLoader(async () => {
22071
- const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-N2DFWWZ3.mjs')]);
22070
+ const [segmentCodec2, code7] = await Promise.all([loadSegmentCodec(), import('./optional-writes-QE7QUI45.mjs')]);
22072
22071
  code7.bindOptionalWritesHost(optionalCodeHost(), segmentCodec2);
22073
22072
  return code7;
22074
22073
  }, "write");
@@ -24291,7 +24290,8 @@ var DURABILITY_CONFIG_KEYS = {
24291
24290
  segmentMaxBytes: true,
24292
24291
  checkpointIntervalMs: true,
24293
24292
  checkpointMutationThreshold: true,
24294
- compactionThreshold: true
24293
+ compactionThreshold: true,
24294
+ rewriteBytesPerCheckpoint: true
24295
24295
  };
24296
24296
  var WORKER_CONFIG_KEYS = {
24297
24297
  enabled: true,
@@ -24702,7 +24702,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
24702
24702
 
24703
24703
  // src/engine/durability-config-code.ts
24704
24704
  var durabilityConfig = codeLoader(
24705
- () => import('./durability-config-XV4XR7Y4.mjs').then((code7) => {
24705
+ () => import('./durability-config-NWU2G2SJ.mjs').then((code7) => {
24706
24706
  code7.bindDurabilityConfigHost(optionalCodeHost());
24707
24707
  return code7;
24708
24708
  }),
@@ -25275,7 +25275,7 @@ var workerCopies = codeLoader(async () => {
25275
25275
  const [segmentCodec2, segmentMerge2, code7] = await Promise.all([
25276
25276
  loadSegmentCodec(),
25277
25277
  loadSegmentMerge(),
25278
- import('./worker-copies-XXX7PF4J.mjs')
25278
+ import('./worker-copies-373P6IYZ.mjs')
25279
25279
  ]);
25280
25280
  code7.bindWorkerCopiesHost(optionalCodeHost(), segmentCodec2, segmentMerge2);
25281
25281
  return code7;
@@ -25521,12 +25521,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
25521
25521
  queueDiskAdoption: orchestrator.queueDiskAdoption,
25522
25522
  awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
25523
25523
  publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
25524
- recordCheckpoint(indexName, documentCount, partitionCount) {
25525
- const entry = indexRegistry.get(indexName);
25526
- if (entry === void 0) return;
25527
- entry.documentCount = documentCount;
25528
- entry.partitionCount = partitionCount;
25529
- },
25530
25524
  engineCore: () => engineCore
25531
25525
  });
25532
25526
  const invalidationCode = config?.invalidation === void 0 ? null : durabilityWiring ?? await loadStorageCode();
@@ -29366,6 +29360,8 @@ function createNarsilFromCore(core2, config) {
29366
29360
  documentCount: manager?.countDocuments() ?? entry.documentCount,
29367
29361
  partitionCount: manager?.partitionCount ?? entry.partitionCount,
29368
29362
  estimatedMemoryBytes: manager?.estimateMemoryBytes() ?? 0,
29363
+ savedBytes: entry.storedSizes?.savedBytes,
29364
+ logBytes: entry.storedSizes?.logBytes,
29369
29365
  language: entry.language.name,
29370
29366
  schema: entry.config.schema,
29371
29367
  heldSearchStates: manager?.heldSearchStates ?? 0
@@ -29373,8 +29369,10 @@ function createNarsilFromCore(core2, config) {
29373
29369
  },
29374
29370
  getPartitionStats(indexName) {
29375
29371
  guardShutdown();
29376
- requireIndex(indexName);
29377
- return executor.getManager(indexName)?.getPartitionStats() ?? [];
29372
+ const entry = requireIndex(indexName);
29373
+ const stats = executor.getManager(indexName)?.getPartitionStats() ?? [];
29374
+ entry.storedSizes?.addTo(stats);
29375
+ return stats;
29378
29376
  },
29379
29377
  insert(indexName, document, docId, options) {
29380
29378
  return withCountedWrite(indexName, () => insertDocument2(mutationCtx, indexName, document, docId, options));
@@ -29595,7 +29593,7 @@ var suggestionsCode = codeLoader(async () => {
29595
29593
  const [msgpack, suggestionStore2, code7] = await Promise.all([
29596
29594
  loadMessagePack(),
29597
29595
  loadSuggestionStore(),
29598
- import('./suggestions-leaf-FXF4AHAY.mjs')
29596
+ import('./suggestions-leaf-ZTU3ZTWJ.mjs')
29599
29597
  ]);
29600
29598
  code7.bindSuggestionsHost(optionalCodeHost(), msgpack, suggestionStore2);
29601
29599
  return code7;
@@ -30749,7 +30747,7 @@ var segmentMerge = codeLoader(async () => {
30749
30747
  const [msgpack, codec, code7] = await Promise.all([
30750
30748
  loadMessagePack(),
30751
30749
  loadSegmentCodec(),
30752
- import('./segment-merge-leaf-FQQDU45E.mjs')
30750
+ import('./segment-merge-leaf-SKMPASFK.mjs')
30753
30751
  ]);
30754
30752
  code7.bindSegmentMergeHost(optionalCodeHost(), msgpack, codec);
30755
30753
  return code7;
@@ -1,5 +1,5 @@
1
- import { getIndexMetadata } from './chunk-XQQKETAF.mjs';
2
- import { fnv1a } from './chunk-2C3BXZGA.mjs';
1
+ import { getIndexMetadata } from './chunk-F2UOTRY7.mjs';
2
+ import { fnv1a } from './chunk-CWXR7OP4.mjs';
3
3
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
4
4
 
5
5
  // src/distribution/cluster-node/routable-allocation.ts
@@ -1,7 +1,7 @@
1
1
  import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-X4UMDSB4.mjs';
2
2
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-JDDXV6UF.mjs';
3
- import { storedStateOf } from './chunk-JCWC75NH.mjs';
4
- import { getLanguage, generateId, decodeSynonymRules } from './chunk-2C3BXZGA.mjs';
3
+ import { storedStateOf } from './chunk-C77OBNPP.mjs';
4
+ import { getLanguage, generateId, decodeSynonymRules } from './chunk-CWXR7OP4.mjs';
5
5
  import { decode } from './chunk-GXCDWLQI.mjs';
6
6
  import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
7
7
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-JP77OWMI.mjs';
@@ -1,7 +1,7 @@
1
1
  import { encodeJson, SYNC_CLOSE_SERVER_ERROR, SYNC_SUBPROTOCOL, encodeScore } from './chunk-FETZYCH7.mjs';
2
- import { answerRefusalOf } from './chunk-Y27E2MGC.mjs';
3
- import { archiveRange } from './chunk-UISV46WX.mjs';
4
- import { readRecentMemoryFigures, readMemoryFigures, DEFAULT_RERANK_WINDOW } from './chunk-2C3BXZGA.mjs';
2
+ import { answerRefusalOf } from './chunk-GGG5R4CU.mjs';
3
+ import { archiveRange } from './chunk-NPXORGMK.mjs';
4
+ import { readRecentMemoryFigures, readMemoryFigures, DEFAULT_RERANK_WINDOW } from './chunk-CWXR7OP4.mjs';
5
5
  import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-LPRRFCVU.mjs';
6
6
  import { MAX_FILTER_DEPTH } from './chunk-YM2HI5N5.mjs';
7
7
  import { bindSignalStop } from './chunk-DG6WIEB3.mjs';
@@ -1,4 +1,4 @@
1
- import { readRecentMemoryFigures, readMemoryFigures } from './chunk-2C3BXZGA.mjs';
1
+ import { readRecentMemoryFigures, readMemoryFigures } from './chunk-CWXR7OP4.mjs';
2
2
  import { collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-LPRRFCVU.mjs';
3
3
  import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
4
4
 
@@ -1,5 +1,5 @@
1
- import { openCountsFile } from './chunk-YXZDTKQS.mjs';
2
- import { SCRAMBLED_ID_BYTES } from './chunk-JCWC75NH.mjs';
1
+ import { openCountsFile } from './chunk-MCAJOLVK.mjs';
2
+ import { SCRAMBLED_ID_BYTES } from './chunk-C77OBNPP.mjs';
3
3
  export { encode } from '@msgpack/msgpack';
4
4
 
5
5
  function bindSuggestionsHost(_host, _msgpack, _suggestionStore) {
@@ -1,10 +1,10 @@
1
- import { syncStorageOf, INDEX_CONFIGURATION_ENVELOPE_VERSION, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration } from './chunk-LOQHA6QK.mjs';
2
- import { archiveLayout } from './chunk-UISV46WX.mjs';
3
- import { manifestFileRefs } from './chunk-KHJPMZE2.mjs';
4
- import { DIRECTORY_IDENTITY_CHECK_INTERVAL_MS } from './chunk-QDZJM7NI.mjs';
1
+ import { syncStorageOf, INDEX_CONFIGURATION_ENVELOPE_VERSION, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration } from './chunk-SA7U67XF.mjs';
2
+ import { archiveLayout } from './chunk-NPXORGMK.mjs';
3
+ import { manifestFileRefs } from './chunk-PK3BGGHO.mjs';
4
+ import { DIRECTORY_IDENTITY_CHECK_INTERVAL_MS } from './chunk-MSIN4NM3.mjs';
5
5
  import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
6
- import { openChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
7
- import { withOpenIndex, emitEngineEvent } from './chunk-2C3BXZGA.mjs';
6
+ import { openChecksummedEnvelope } from './chunk-C77OBNPP.mjs';
7
+ import { withOpenIndex, emitEngineEvent } from './chunk-CWXR7OP4.mjs';
8
8
  import { decode } from './chunk-GXCDWLQI.mjs';
9
9
  import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
10
10
  import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';