@delali/narsil 0.3.1-next.185 → 0.3.1-next.193

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +1 -0
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  7. package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
  8. package/dist/browser/{chunk-CPSHS46N.mjs → chunk-UQG4HUGQ.mjs} +82 -50
  9. package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
  10. package/dist/browser/{module-XVLHJ7T7.mjs → module-ZZTNG5JZ.mjs} +26 -3
  11. package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
  12. package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-N3JMQ6CF.mjs} +14569 -13747
  13. package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
  14. package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
  15. package/dist/{chunk-CODGHJVW.mjs → chunk-2HVQYOOD.mjs} +40 -12
  16. package/dist/{chunk-M5I64AQW.mjs → chunk-3N5GUXBI.mjs} +33 -33
  17. package/dist/{chunk-DU23J5JG.mjs → chunk-5CIHQIA4.mjs} +6 -6
  18. package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-62X472AC.mjs} +1 -1
  19. package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
  20. package/dist/{chunk-G3XRHAJA.mjs → chunk-6MC6BCXK.mjs} +3 -3
  21. package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
  22. package/dist/{chunk-MHBG33R5.mjs → chunk-7X4IIREW.mjs} +76 -46
  23. package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
  24. package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
  25. package/dist/{chunk-LR6JJVML.mjs → chunk-D2EJUM4A.mjs} +764 -699
  26. package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
  27. package/dist/{chunk-2YWWHMVC.mjs → chunk-FGOGT273.mjs} +2 -2
  28. package/dist/{chunk-5BUEBIXR.mjs → chunk-GLXLSPLO.mjs} +18 -5
  29. package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
  30. package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
  31. package/dist/{chunk-BVZWWMOU.mjs → chunk-HHYQMGPW.mjs} +2 -2
  32. package/dist/{chunk-SL53DHVE.mjs → chunk-HSA3H3VW.mjs} +121 -68
  33. package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
  34. package/dist/{chunk-JDG6CCXM.mjs → chunk-JENWA4WX.mjs} +2 -2
  35. package/dist/{chunk-2GONI5HL.mjs → chunk-JXINPXRS.mjs} +5 -4
  36. package/dist/chunk-K5GNK5GE.mjs +602 -0
  37. package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
  38. package/dist/{chunk-EDEE7ETQ.mjs → chunk-KOA6T2IC.mjs} +3 -3
  39. package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
  40. package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
  41. package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
  42. package/dist/{chunk-VYSWVQCE.mjs → chunk-PAPQGC6I.mjs} +2 -2
  43. package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
  44. package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
  45. package/dist/{chunk-FBQYC5OX.mjs → chunk-SB5HCIIO.mjs} +16 -13
  46. package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
  47. package/dist/{chunk-O337MDRJ.mjs → chunk-SHES4VP5.mjs} +38 -13
  48. package/dist/{chunk-H4QYQ5DE.mjs → chunk-SIIDIGKI.mjs} +2 -2
  49. package/dist/{chunk-A3OAFUZI.mjs → chunk-SM2SYVIX.mjs} +2 -2
  50. package/dist/{chunk-II6XF6M4.mjs → chunk-SMLUZGSI.mjs} +9 -8
  51. package/dist/{chunk-2KCCQK4R.mjs → chunk-TSE2VWPX.mjs} +2 -2
  52. package/dist/{chunk-WSIU6YXQ.mjs → chunk-YPEJ6ODL.mjs} +1732 -743
  53. package/dist/{chunk-7WXGUTWK.mjs → chunk-Z6SIJH5O.mjs} +17 -6
  54. package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
  55. package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
  56. package/dist/{chunk-FCQMZDRI.mjs → chunk-ZVZYJIIP.mjs} +3 -3
  57. package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
  58. package/dist/client.d.ts +7 -7
  59. package/dist/client.mjs +3 -1265
  60. package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
  61. package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
  62. package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
  63. package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-WSZALYCV.mjs} +8 -8
  64. package/dist/distribution/coordinator/etcd.d.ts +1 -1
  65. package/dist/distribution/coordinator/etcd.mjs +35 -15
  66. package/dist/distribution/coordinator/in-memory.d.ts +1 -1
  67. package/dist/distribution/coordinator/in-memory.mjs +16 -6
  68. package/dist/distribution/transport/grpc.mjs +2 -2
  69. package/dist/distribution/transport/tcp.mjs +2 -2
  70. package/dist/distribution.d.ts +70 -9
  71. package/dist/distribution.mjs +1695 -680
  72. package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
  73. package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
  74. package/dist/embeddings/openai.mjs +2 -2
  75. package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
  76. package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
  77. package/dist/index.browser.mjs +3 -3
  78. package/dist/index.d.ts +4 -4
  79. package/dist/index.mjs +8 -8
  80. package/dist/index.react-native.mjs +9 -9
  81. package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-N2DNPEA3.mjs} +21 -21
  82. package/dist/{module-Z3CL4GJ4.mjs → module-UQFIJQMG.mjs} +33 -13
  83. package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
  84. package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
  85. package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-CGDBJBHQ.mjs} +31 -18
  86. package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-UK44CVXG.mjs} +20 -20
  87. package/dist/partial-words.browser.mjs +3 -3
  88. package/dist/partial-words.mjs +8 -8
  89. package/dist/pattern.browser.mjs +2 -2
  90. package/dist/pattern.mjs +3 -3
  91. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
  92. package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
  93. package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
  94. package/dist/{query-actions-7EOKT3WD.mjs → query-actions-7BL3YZDW.mjs} +8 -8
  95. package/dist/query-suggestions/merge-worker.mjs +11 -11
  96. package/dist/query-suggestions.browser.mjs +3 -3
  97. package/dist/query-suggestions.d.ts +1 -1
  98. package/dist/query-suggestions.mjs +9 -9
  99. package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
  100. package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-63DZ26X5.mjs} +277 -75
  101. package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-BNFGA7TY.mjs} +3 -3
  103. package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
  104. package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
  105. package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
  106. package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-IP3QLETQ.mjs} +5 -5
  107. package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-LOKGE4OQ.mjs} +3 -3
  108. package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
  109. package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
  110. package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-UYAZLGEM.mjs} +1 -1
  111. package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
  112. package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-XHYOQ4K4.mjs} +2 -2
  113. package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
  114. package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
  115. package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
  116. package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
  117. package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
  118. package/dist/react-native/chunks/{module-COLNJNML.mjs → module-QRCYB7QV.mjs} +32 -9
  119. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  120. package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
  121. package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-GC6JWMAF.mjs} +31 -18
  122. package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
  123. package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
  124. package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-2ACONS4W.mjs} +12 -12
  125. package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-SSIWXVUG.mjs} +10 -10
  126. package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-I2WDFE6O.mjs} +11 -11
  127. package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-YXXMG4VR.mjs} +14727 -13915
  128. package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-FS7A4PZA.mjs} +11 -11
  129. package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-PSFCT5EX.mjs} +11 -11
  130. package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-QRMISMEL.mjs} +10 -10
  131. package/dist/react-native/partial-words.mjs +9 -9
  132. package/dist/react-native/pattern.mjs +5 -5
  133. package/dist/react-native/query-suggestions.mjs +9 -9
  134. package/dist/react-native/sync.mjs +433 -51
  135. package/dist/react-native/synonyms.mjs +9 -9
  136. package/dist/react.d.ts +8 -8
  137. package/dist/react.mjs +7 -0
  138. package/dist/{rerank-KTV5L7MZ.mjs → rerank-4GFC26FD.mjs} +8 -8
  139. package/dist/rerankers/http.mjs +2 -2
  140. package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
  141. package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-E4EIUYVV.mjs} +11 -11
  142. package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-MP6DILVE.mjs} +9 -9
  143. package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-ED2NMUC5.mjs} +9 -9
  144. package/dist/server.d.ts +13 -9
  145. package/dist/server.mjs +875 -348
  146. package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-HQFO37W7.mjs} +5234 -5835
  147. package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-PL2G7RM5.mjs} +10 -10
  148. package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-I3MEHACP.mjs} +12 -12
  149. package/dist/sync/filesystem.d.ts +2 -2
  150. package/dist/sync/filesystem.mjs +16 -16
  151. package/dist/sync/indexeddb.browser.mjs +1 -1
  152. package/dist/sync/indexeddb.d.ts +2 -2
  153. package/dist/sync/indexeddb.mjs +3 -3
  154. package/dist/sync.browser.mjs +35 -1108
  155. package/dist/sync.d.ts +3 -3
  156. package/dist/sync.mjs +430 -51
  157. package/dist/synonyms.browser.mjs +3 -3
  158. package/dist/synonyms.mjs +8 -8
  159. package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
  160. package/dist/thread-LTFI3RK5.mjs +31 -0
  161. package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
  162. package/dist/vector/search-worker.mjs +5 -5
  163. package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-P7YWXEP5.mjs} +11 -11
  164. package/dist/workers/entry.mjs +14 -14
  165. package/openapi.json +144 -0
  166. package/package.json +5 -7
  167. package/dist/chunk-URVH6BLO.mjs +0 -217
  168. package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
  169. package/dist/thread-NQRNW3IH.mjs +0 -31
  170. /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
@@ -1,22 +1,545 @@
1
- import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL, resumePagedFile, suspendPagedFile, cachedFileBytes, requireNativePage, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, CACHED_DOCUMENT_BLOCK, failNativePage, readFileRangeLater, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-M5I64AQW.mjs';
1
+ import { PAGED_SEGMENT_ENVELOPE_VERSION, loadFailed, checkEverySectionPage, openPagedFile, sectionNamed, createSectionReader, closePagedFile, readWholeSectionChecked, readCacheOfThisThread, createReadCache, MERGE_READ_CACHE_BYTES, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, requireNativePage, ID_SAMPLE_INTERVAL, SMALLEST_BLOCK_POSTINGS, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, resumePagedFile, suspendPagedFile, cachedFileBytes, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, readFileRangeLater, CACHED_DOCUMENT_BLOCK, failNativePage, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-3N5GUXBI.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
3
3
  import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
4
- import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-2KCCQK4R.mjs';
4
+ import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-TSE2VWPX.mjs';
5
5
  import { DEFAULT_REWRITE_BYTES_PER_CHECKPOINT, DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, REWRITE_ABOVE_REMOVED_DOCUMENT_SHARE, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-MSIN4NM3.mjs';
6
- import { encode } from './chunk-CV3Q2EKO.mjs';
7
- import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
8
- import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-EDEE7ETQ.mjs';
9
- import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-SL53DHVE.mjs';
10
- import { encodeDocument } from './chunk-KLVMK3R5.mjs';
6
+ import { encode } from './chunk-S5QMWECL.mjs';
7
+ import { restoreVectorFields } from './chunk-R2PJKSNL.mjs';
8
+ import { requireWithinFileLimit, HEADER_SIZE, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, readHeader, packChecksummedEnvelope } from './chunk-KOA6T2IC.mjs';
9
+ import { decodeStoredDocument, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-HSA3H3VW.mjs';
10
+ import { encodeDocument } from './chunk-2CKDCC65.mjs';
11
11
  import { decode } from './chunk-GXCDWLQI.mjs';
12
- import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-X5NHWOXT.mjs';
12
+ import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-7P5YWDRQ.mjs';
13
13
  import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
14
- import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
14
+ import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
15
15
  import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
16
16
  import { deflateRawSync, inflateRawSync } from 'zlib';
17
17
 
18
+ // src/persistence/paged/verify.ts
19
+ async function orLoadFailure(key, read) {
20
+ try {
21
+ return await read();
22
+ } catch (error) {
23
+ if (error instanceof NarsilError) throw error;
24
+ return loadFailed(
25
+ `The segment "${key}" cannot be read: ${error instanceof Error ? error.message : String(error)}`,
26
+ {
27
+ key
28
+ }
29
+ );
30
+ }
31
+ }
32
+ async function envelopeVersionOf(directory, key) {
33
+ const { readRange } = directory;
34
+ if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
35
+ const bytes = await orLoadFailure(key, () => readRange.call(directory, key, 4, 1));
36
+ return bytes[0];
37
+ }
38
+ async function isPagedSegmentFile(directory, key) {
39
+ return await envelopeVersionOf(directory, key) === PAGED_SEGMENT_ENVELOPE_VERSION;
40
+ }
41
+ async function requirePagedPayloadChecksum(directory, listing) {
42
+ const { readRange } = directory;
43
+ const { key, bytes } = listing;
44
+ if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
45
+ const header2 = readHeader(await readRange.call(directory, key, 0, HEADER_SIZE));
46
+ if (HEADER_SIZE + header2.payloadLength !== bytes) {
47
+ loadFailed(`The segment "${key}" holds ${bytes} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`);
48
+ }
49
+ await requirePayloadChecksum(key, header2, (offset, length) => readRange.call(directory, key, offset, length));
50
+ }
51
+ async function requirePagedPathChecksum(path) {
52
+ const read = async (offset, length) => {
53
+ const bytes = new Uint8Array(length);
54
+ if (!await readFileRangeLater(path, offset, bytes))
55
+ loadFailed(`The segment "${path}" cannot be read at ${offset}`);
56
+ return bytes;
57
+ };
58
+ await requirePayloadChecksum(path, readHeader(await read(0, HEADER_SIZE)), read);
59
+ }
60
+ async function requirePayloadChecksum(key, header2, read) {
61
+ const bytes = HEADER_SIZE + header2.payloadLength;
62
+ let state = crc32Init();
63
+ for (let at = HEADER_SIZE; at < bytes; at += FILE_ASSEMBLY_CHUNK_BYTES) {
64
+ const size = Math.min(FILE_ASSEMBLY_CHUNK_BYTES, bytes - at);
65
+ state = crc32Update(state, await read(at, size));
66
+ }
67
+ const computed = crc32Final(state);
68
+ if (computed !== header2.checksum) {
69
+ throw new NarsilError(
70
+ ErrorCodes.PERSISTENCE_CRC_MISMATCH,
71
+ `The checksum of the segment "${key}" does not match its payload`,
72
+ { key, expected: header2.checksum, computed }
73
+ );
74
+ }
75
+ }
76
+ async function pagedListingOf(directory, ref) {
77
+ const { byteLength } = directory;
78
+ const length = byteLength === void 0 ? ref.bytes : await byteLength.call(directory, ref.key);
79
+ if (length === null) loadFailed(`The segment "${ref.key}" that the manifest lists is missing`, { key: ref.key });
80
+ if (length !== ref.bytes) {
81
+ loadFailed(`The segment "${ref.key}" holds ${length} bytes where the manifest lists ${ref.bytes}`, {
82
+ key: ref.key,
83
+ bytes: length,
84
+ listed: ref.bytes
85
+ });
86
+ }
87
+ return { key: ref.key, path: await directory.pathOf(ref.key), bytes: ref.bytes, checksum: ref.checksum };
88
+ }
89
+
90
+ // src/persistence/paged/totals.ts
91
+ var VALUE_BYTES = Float64Array.BYTES_PER_ELEMENT;
92
+ function readPagedTotals(file, totals, first, output, fault) {
93
+ const section = totals.native;
94
+ if (section !== null && file.native !== null) {
95
+ const status = file.native.core.readPagedTotals(
96
+ section,
97
+ file.cache.pageCache(),
98
+ first,
99
+ output.length,
100
+ output,
101
+ fault
102
+ );
103
+ requireNativePage(status, file.path, "tokenTotals", fault[2]);
104
+ return;
105
+ }
106
+ const bytes = totals.bytesAt(first * VALUE_BYTES, output.byteLength);
107
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
108
+ for (let at = 0; at < output.length; at++) {
109
+ const value = view.getFloat64(at * VALUE_BYTES, true);
110
+ if (!Number.isFinite(value) || value < 0) loadFailed(`The term total at ${first + at} of "${file.path}" is invalid`);
111
+ }
112
+ for (let at = 0; at < output.length; at++) output[at] = view.getFloat64(at * VALUE_BYTES, true);
113
+ }
114
+
115
+ // src/persistence/paged/lists.ts
116
+ var LITTLE_ENDIAN_PLATFORM = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
117
+ function swappedInPlace(bytes, width) {
118
+ for (let at = 0; at + width <= bytes.length; at += width) {
119
+ for (let low = 0, high = width - 1; low < high; low++, high--) {
120
+ const held = bytes[at + low];
121
+ bytes[at + low] = bytes[at + high];
122
+ bytes[at + high] = held;
123
+ }
124
+ }
125
+ }
126
+ function fromLittleEndian(bytes, width) {
127
+ if (!LITTLE_ENDIAN_PLATFORM) swappedInPlace(bytes, width);
128
+ }
129
+ function requireListRules(list, snapshot, token) {
130
+ const { docIds, fieldNameIndices } = list;
131
+ const documents = snapshot.documentCount;
132
+ const fields = snapshot.fieldNames.length;
133
+ for (let row = 0; row < docIds.length; row++) {
134
+ if (docIds[row] >= documents) {
135
+ loadFailed(`Token ${token} of "${snapshot.key}" names document ${docIds[row]} of ${documents}`, { token, row });
136
+ }
137
+ if (fieldNameIndices[row] >= fields) {
138
+ loadFailed(`Token ${token} of "${snapshot.key}" names field ${fieldNameIndices[row]} of ${fields}`, {
139
+ token,
140
+ row
141
+ });
142
+ }
143
+ if (row === 0) continue;
144
+ const before = docIds[row - 1];
145
+ if (docIds[row] < before || docIds[row] === before && fieldNameIndices[row] <= fieldNameIndices[row - 1]) {
146
+ loadFailed(`The list of token ${token} of "${snapshot.key}" descends at posting ${row}`, { token, row });
147
+ }
148
+ }
149
+ }
150
+ function createListReader(file, snapshot, sectionReader) {
151
+ const { postingOffsets } = snapshot;
152
+ const postings = sectionReader("postings");
153
+ const positionOffsets = sectionReader("positionOffsets");
154
+ const positions = sectionReader("positions");
155
+ const totals = sectionReader("tokenTotals");
156
+ const { cache, fileNumber } = file;
157
+ const fault = new BigUint64Array(3);
158
+ const total = new Float64Array(1);
159
+ function listAt(token) {
160
+ const held = cache.get(fileNumber, CACHED_LIST, token);
161
+ if (held !== void 0) return held;
162
+ if (postings === null) return loadFailed(`The segment "${snapshot.key}" holds no postings`);
163
+ const start = postingOffsets[token];
164
+ const count = postingOffsets[token + 1] - start;
165
+ const buffer = new ArrayBuffer(count * POSTING_WIDTH_BYTES);
166
+ const bytes = new Uint8Array(buffer);
167
+ const descriptor = postings.native;
168
+ if (descriptor !== null && file.native !== null) {
169
+ const status = file.native.core.readPagedPostings(
170
+ descriptor,
171
+ cache.pageCache(),
172
+ start,
173
+ count,
174
+ snapshot.documentCount,
175
+ snapshot.fieldNames.length,
176
+ bytes,
177
+ fault
178
+ );
179
+ requireNativePage(status, file.path, "postings", fault[2]);
180
+ } else {
181
+ postings.read(start * POSTING_WIDTH_BYTES, bytes);
182
+ }
183
+ if (!LITTLE_ENDIAN_PLATFORM) {
184
+ swappedInPlace(bytes.subarray(0, count * 4), 4);
185
+ swappedInPlace(bytes.subarray(count * 4), 2);
186
+ }
187
+ const list = {
188
+ docIds: new Uint32Array(buffer, 0, count),
189
+ termFrequencies: new Uint16Array(buffer, count * 4, count),
190
+ fieldNameIndices: new Uint16Array(buffer, count * 6, count)
191
+ };
192
+ if (descriptor === null || file.native === null) requireListRules(list, snapshot, token);
193
+ cache.put(fileNumber, CACHED_LIST, token, list, buffer.byteLength);
194
+ return list;
195
+ }
196
+ function positionsAt(token) {
197
+ if (positionOffsets === null || positions === null) return null;
198
+ const held = cache.get(fileNumber, CACHED_POSITIONS, token);
199
+ if (held !== void 0) return held;
200
+ const start = postingOffsets[token];
201
+ const count = postingOffsets[token + 1] - start;
202
+ const offsetBytes = new Uint8Array((count + 1) * POSITION_WIDTH_BYTES);
203
+ positionOffsets.read(start * POSITION_WIDTH_BYTES, offsetBytes);
204
+ fromLittleEndian(offsetBytes, POSITION_WIDTH_BYTES);
205
+ const absolute = new Uint32Array(offsetBytes.buffer);
206
+ const first = absolute[0];
207
+ const end = absolute[count];
208
+ if (end < first || end * POSITION_WIDTH_BYTES > positions.length || start === 0 && first !== 0 || (start + count + 1) * POSITION_WIDTH_BYTES === positionOffsets.length && end * POSITION_WIDTH_BYTES !== positions.length)
209
+ loadFailed(`The positions of token ${token} of "${snapshot.key}" leave their section`, { token });
210
+ const offsets = new Uint32Array(count + 1);
211
+ const valueBytes = new Uint8Array((end - first) * POSITION_WIDTH_BYTES);
212
+ const loaded = { offsets, values: new Uint32Array(valueBytes.buffer) };
213
+ const offsetSection = positionOffsets.native;
214
+ const valueSection = positions.native;
215
+ if (offsetSection !== null && valueSection !== null && file.native !== null) {
216
+ const status = file.native.core.readPagedPositions(
217
+ offsetSection,
218
+ valueSection,
219
+ cache.pageCache(),
220
+ start,
221
+ count,
222
+ offsets,
223
+ loaded.values,
224
+ fault
225
+ );
226
+ requireNativePage(
227
+ status,
228
+ file.path,
229
+ fault[1] === offsetSection.offset ? "positionOffsets" : "positions",
230
+ fault[2]
231
+ );
232
+ } else {
233
+ for (let row = 0; row <= count; row++) {
234
+ if (absolute[row] < first || absolute[row] > end || row > 0 && absolute[row] < absolute[row - 1]) {
235
+ loadFailed(`The position offsets of token ${token} of "${snapshot.key}" leave their list`, { token, row });
236
+ }
237
+ offsets[row] = absolute[row] - first;
238
+ }
239
+ positions.read(first * POSITION_WIDTH_BYTES, valueBytes);
240
+ fromLittleEndian(valueBytes, POSITION_WIDTH_BYTES);
241
+ for (let row = 0; row < count; row++) {
242
+ for (let at = offsets[row] + 1; at < offsets[row + 1]; at++) {
243
+ if (loaded.values[at] < loaded.values[at - 1])
244
+ loadFailed(`The positions of token ${token} of "${snapshot.key}" descend`, { token, row });
245
+ }
246
+ }
247
+ }
248
+ cache.put(fileNumber, CACHED_POSITIONS, token, loaded, offsetBytes.byteLength + valueBytes.byteLength);
249
+ return loaded;
250
+ }
251
+ function totalAt(token) {
252
+ if (totals === null) return loadFailed(`The segment "${snapshot.key}" holds no token totals`);
253
+ readPagedTotals(file, totals, token, total, fault);
254
+ return total[0];
255
+ }
256
+ function listedIndexOf(token) {
257
+ const { blockLists } = snapshot;
258
+ let low = 0;
259
+ let high = blockLists.length;
260
+ while (low < high) {
261
+ const middle = low + high >>> 1;
262
+ if (blockLists[middle] < token) low = middle + 1;
263
+ else high = middle;
264
+ }
265
+ return low < blockLists.length && blockLists[low] === token ? low : -1;
266
+ }
267
+ function storedBoundsAt(token) {
268
+ const listed = listedIndexOf(token);
269
+ if (listed < 0) return null;
270
+ const first = snapshot.blockListOffsets[listed];
271
+ const blockCount = snapshot.blockListOffsets[listed + 1] - first;
272
+ const { blockSummaries } = snapshot;
273
+ const view = new DataView(blockSummaries.buffer, blockSummaries.byteOffset, blockSummaries.byteLength);
274
+ const bounds = {
275
+ structureRevision: 0,
276
+ blockCount,
277
+ entryEnd: new Int32Array(blockCount),
278
+ maxTermFrequency: new Int32Array(blockCount),
279
+ minFieldLength: new Float64Array(blockCount),
280
+ maxEntriesPerDocument: new Int32Array(blockCount),
281
+ documentCount: new Int32Array(blockCount)
282
+ };
283
+ const length = postingOffsets[token + 1] - postingOffsets[token];
284
+ for (let block = 0; block < blockCount; block++) {
285
+ const place = (first + block) * BLOCK_SUMMARY_BYTES;
286
+ bounds.entryEnd[block] = view.getUint32(place, true);
287
+ bounds.maxTermFrequency[block] = view.getUint16(place + 4, true);
288
+ bounds.minFieldLength[block] = view.getFloat64(place + 6, true);
289
+ bounds.maxEntriesPerDocument[block] = view.getUint32(place + 14, true);
290
+ bounds.documentCount[block] = view.getUint32(place + 18, true);
291
+ const before = block === 0 ? 0 : bounds.entryEnd[block - 1];
292
+ if (bounds.entryEnd[block] <= before || bounds.entryEnd[block] > length) {
293
+ loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" ends outside its list`, { token, block });
294
+ }
295
+ }
296
+ if (blockCount === 0 || bounds.entryEnd[blockCount - 1] !== length) {
297
+ loadFailed(`The blocks of token ${token} of "${snapshot.key}" fail to cover its list`, { token });
298
+ }
299
+ return bounds;
300
+ }
301
+ return { listAt, positionsAt, totalAt, storedBoundsAt };
302
+ }
303
+ function createPagedPostingViews(snapshot, lists, tombstones, counts) {
304
+ const { postingOffsets, tracksPositions } = snapshot;
305
+ const views = /* @__PURE__ */ new Map();
306
+ function buildView(token) {
307
+ const length = postingOffsets[token + 1] - postingOffsets[token];
308
+ return {
309
+ length,
310
+ get docIds() {
311
+ return lists.listAt(token).docIds;
312
+ },
313
+ get termFrequencies() {
314
+ return lists.listAt(token).termFrequencies;
315
+ },
316
+ get fieldNameIndices() {
317
+ return lists.listAt(token).fieldNameIndices;
318
+ },
319
+ get positions() {
320
+ const loaded = lists.positionsAt(token);
321
+ if (loaded === null) return null;
322
+ const rows = new Array(length);
323
+ for (let row = 0; row < length; row++) {
324
+ rows[row] = [...loaded.values.subarray(loaded.offsets[row], loaded.offsets[row + 1])];
325
+ }
326
+ return rows;
327
+ },
328
+ holdsPositions: tracksPositions,
329
+ positionCountAt(row) {
330
+ const loaded = lists.positionsAt(token);
331
+ return loaded === null ? 0 : loaded.offsets[row + 1] - loaded.offsets[row];
332
+ },
333
+ get positionStore() {
334
+ return lists.positionsAt(token)?.values ?? null;
335
+ },
336
+ positionStartAt(row) {
337
+ return lists.positionsAt(token)?.offsets[row] ?? 0;
338
+ },
339
+ docIdSet: {
340
+ get size() {
341
+ return counts.documentFrequency(token);
342
+ }
343
+ },
344
+ deletedDocs: tombstones,
345
+ get totalTermFrequency() {
346
+ return lists.totalAt(token) - counts.removedOccurrences(token);
347
+ },
348
+ get structureRevision() {
349
+ return tombstones.revision;
350
+ },
351
+ ordered: true,
352
+ storedBlockBounds() {
353
+ return tombstones.size === 0 ? lists.storedBoundsAt(token) : null;
354
+ }
355
+ };
356
+ }
357
+ return {
358
+ viewAt(payloadSlot) {
359
+ let view = views.get(payloadSlot);
360
+ if (view === void 0) {
361
+ if (views.size >= LIST_VIEWS_HELD_PER_SEGMENT) views.clear();
362
+ view = buildView(payloadSlot);
363
+ views.set(payloadSlot, view);
364
+ }
365
+ return view;
366
+ },
367
+ viewReadOnce(payloadSlot) {
368
+ return buildView(payloadSlot);
369
+ }
370
+ };
371
+ }
372
+
373
+ // src/persistence/paged/removals.ts
374
+ var VALUE_BYTES2 = 4;
375
+ function createRemovalReader(file, removalOffsets, removals, tokenCount) {
376
+ const fault = new BigUint64Array(3);
377
+ return (ordinal) => {
378
+ const bounds = removalOffsets.bytesAt(ordinal * VALUE_BYTES2, VALUE_BYTES2 * 2);
379
+ const boundsView = new DataView(bounds.buffer, bounds.byteOffset, bounds.byteLength);
380
+ const from = boundsView.getUint32(0, true);
381
+ const to = boundsView.getUint32(VALUE_BYTES2, true);
382
+ if (to < from || to * VALUE_BYTES2 > removals.length || from % 2 !== 0 || to % 2 !== 0 || ordinal === 0 && from !== 0 || (ordinal + 2) * VALUE_BYTES2 === removalOffsets.length && to * VALUE_BYTES2 !== removals.length) {
383
+ loadFailed(`The removal entry of document ${ordinal} of "${file.path}" leaves its list`, { ordinal });
384
+ }
385
+ const values = new Uint32Array(to - from);
386
+ const offsetSection = removalOffsets.native;
387
+ const valueSection = removals.native;
388
+ if (offsetSection !== null && valueSection !== null && file.native !== null) {
389
+ const status = file.native.core.readPagedRemovals(
390
+ offsetSection,
391
+ valueSection,
392
+ file.cache.pageCache(),
393
+ ordinal,
394
+ tokenCount,
395
+ values,
396
+ fault
397
+ );
398
+ if (status < 0)
399
+ requireNativePage(
400
+ status,
401
+ file.path,
402
+ fault[1] === offsetSection.offset ? "removalOffsets" : "removals",
403
+ fault[2]
404
+ );
405
+ } else {
406
+ const entry = removals.bytesAt(from * VALUE_BYTES2, values.byteLength);
407
+ const view = new DataView(entry.buffer, entry.byteOffset, entry.byteLength);
408
+ let previous = -1;
409
+ for (let at = 0; at < values.length; at += 2) {
410
+ const token = view.getUint32(at * VALUE_BYTES2, true);
411
+ if (token >= tokenCount || token <= previous) {
412
+ loadFailed(`The removal entry of document ${ordinal} of "${file.path}" holds invalid token ${token}`, {
413
+ ordinal
414
+ });
415
+ }
416
+ previous = token;
417
+ values[at] = token;
418
+ values[at + 1] = view.getUint32((at + 1) * VALUE_BYTES2, true);
419
+ }
420
+ }
421
+ return values;
422
+ };
423
+ }
424
+
425
+ // src/persistence/paged/check-contents.ts
426
+ function requireIds(segment, snapshot) {
427
+ const ids = segment.arrays.idTable;
428
+ let previous = null;
429
+ for (let position = 0; position < snapshot.documentCount; position++) {
430
+ const id = ids.idAt(ids.sortedOrdinalAt(position));
431
+ if (previous !== null && compareCodePoints(previous, id) >= 0)
432
+ loadFailed(`The ID order of "${snapshot.key}" does not ascend at position ${position}`, { position });
433
+ if (position % ID_SAMPLE_INTERVAL === 0 && snapshot.idSampleIds[position / ID_SAMPLE_INTERVAL] !== id)
434
+ loadFailed(`The ID sample of "${snapshot.key}" differs at position ${position}`, { position });
435
+ previous = id;
436
+ }
437
+ }
438
+ function requirePostings(lists, snapshot) {
439
+ const fields = snapshot.fieldNames.map((field) => {
440
+ const at = snapshot.fieldLengthNames.indexOf(field);
441
+ return at < 0 ? null : snapshot.fieldLengthColumns[at];
442
+ });
443
+ for (let token = 0; token < snapshot.tokenTable.documentFrequencies.length; token++) {
444
+ const list = lists.listAt(token);
445
+ lists.positionsAt(token);
446
+ const bounds = lists.storedBoundsAt(token);
447
+ if (list.docIds.length > SMALLEST_BLOCK_POSTINGS !== (bounds !== null))
448
+ loadFailed(`The block summaries of token ${token} of "${snapshot.key}" do not cover its postings`, { token });
449
+ let documents = 0;
450
+ let occurrences = 0;
451
+ let block = 0;
452
+ let start = 0;
453
+ let maxFrequency = 0;
454
+ let minLength = Number.POSITIVE_INFINITY;
455
+ let maxEntries = 0;
456
+ let blockDocuments = 0;
457
+ let row = 0;
458
+ while (row < list.docIds.length) {
459
+ const document = list.docIds[row];
460
+ let entries = 0;
461
+ do {
462
+ const frequency = list.termFrequencies[row];
463
+ occurrences += frequency;
464
+ maxFrequency = Math.max(maxFrequency, frequency);
465
+ minLength = Math.min(minLength, fields[list.fieldNameIndices[row]]?.[document] ?? 0);
466
+ row++;
467
+ entries++;
468
+ } while (row < list.docIds.length && list.docIds[row] === document);
469
+ documents++;
470
+ blockDocuments++;
471
+ maxEntries = Math.max(maxEntries, entries);
472
+ if (bounds === null || row - start < SMALLEST_BLOCK_POSTINGS && row < list.docIds.length) continue;
473
+ if (block >= bounds.blockCount || bounds.entryEnd[block] !== row || bounds.maxTermFrequency[block] !== maxFrequency || bounds.minFieldLength[block] !== minLength || bounds.maxEntriesPerDocument[block] !== maxEntries || bounds.documentCount[block] !== blockDocuments)
474
+ loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" differs from its postings`, { token, block });
475
+ block++;
476
+ start = row;
477
+ maxFrequency = 0;
478
+ minLength = Number.POSITIVE_INFINITY;
479
+ maxEntries = 0;
480
+ blockDocuments = 0;
481
+ }
482
+ if (documents !== snapshot.tokenTable.documentFrequencies[token] || occurrences !== lists.totalAt(token))
483
+ loadFailed(`The counts of token ${token} of "${snapshot.key}" differ from its postings`, { token });
484
+ }
485
+ }
486
+ async function checkPagedContents(segment, cache) {
487
+ const { paged } = segment;
488
+ if (paged === null) return;
489
+ const { snapshot } = paged;
490
+ for (const section of snapshot.sections) await checkEverySectionPage(snapshot.path, section, () => true);
491
+ const file = openPagedFile(snapshot.path, cache);
492
+ try {
493
+ const sectionReader = (name) => {
494
+ const section = sectionNamed(snapshot.sections, name);
495
+ return section === void 0 ? null : createSectionReader(file, section);
496
+ };
497
+ const required = (name) => {
498
+ const section = sectionNamed(snapshot.sections, name);
499
+ if (section === void 0) loadFailed(`The segment "${snapshot.key}" holds no section "${name}"`);
500
+ return createSectionReader(file, section);
501
+ };
502
+ const removals = createRemovalReader(
503
+ file,
504
+ required("removalOffsets"),
505
+ required("removals"),
506
+ segment.arrays.tokenTable.size
507
+ );
508
+ for (let ordinal = 0; ordinal < snapshot.documentCount; ordinal++) {
509
+ decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields);
510
+ paged.writeRecordAt(ordinal);
511
+ removals(ordinal);
512
+ }
513
+ if (snapshot.documentCount === 0) {
514
+ for (const name of ["idOffsets", "removalOffsets"]) {
515
+ const offsets = required(name).bytesAt(0, 4);
516
+ if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
517
+ loadFailed(`Section "${name}" of "${snapshot.key}" does not start at zero`);
518
+ }
519
+ }
520
+ if (snapshot.tracksPositions && snapshot.postingOffsets[snapshot.postingOffsets.length - 1] === 0) {
521
+ const offsets = required("positionOffsets").bytesAt(0, 4);
522
+ if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
523
+ loadFailed(`The position offsets of "${snapshot.key}" do not start at zero`);
524
+ }
525
+ requireIds(segment, snapshot);
526
+ requirePostings(createListReader(file, snapshot, sectionReader), snapshot);
527
+ segment.arrays.readSurfaceForms();
528
+ } finally {
529
+ closePagedFile(file);
530
+ }
531
+ }
532
+ async function checkPagedSelections(selections) {
533
+ for (const { segment } of selections) {
534
+ const { paged } = segment;
535
+ if (paged === null || paged.payloadChecked) continue;
536
+ await requirePagedPathChecksum(paged.snapshot.path);
537
+ paged.payloadChecked = true;
538
+ }
539
+ }
540
+
18
541
  // src/serialization/keyword-segment/little-endian.ts
19
- var LITTLE_ENDIAN_PLATFORM = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
542
+ var LITTLE_ENDIAN_PLATFORM2 = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
20
543
  function swappedCopy(values) {
21
544
  const width = values.BYTES_PER_ELEMENT;
22
545
  const source = new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
@@ -27,7 +550,7 @@ function swappedCopy(values) {
27
550
  return copy;
28
551
  }
29
552
  function littleEndianBytes(values) {
30
- if (!LITTLE_ENDIAN_PLATFORM) return swappedCopy(values);
553
+ if (!LITTLE_ENDIAN_PLATFORM2) return swappedCopy(values);
31
554
  return new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
32
555
  }
33
556
  function allocate(byteLength, shared) {
@@ -36,7 +559,7 @@ function allocate(byteLength, shared) {
36
559
  function copyInto(bytes, width, shared) {
37
560
  const buffer = allocate(bytes.byteLength, shared);
38
561
  const target = new Uint8Array(buffer);
39
- if (LITTLE_ENDIAN_PLATFORM) {
562
+ if (LITTLE_ENDIAN_PLATFORM2) {
40
563
  target.set(bytes);
41
564
  return buffer;
42
565
  }
@@ -725,31 +1248,6 @@ function residentBytesOf(snapshot) {
725
1248
  return bytes;
726
1249
  }
727
1250
 
728
- // src/persistence/paged/totals.ts
729
- var VALUE_BYTES = Float64Array.BYTES_PER_ELEMENT;
730
- function readPagedTotals(file, totals, first, output, fault) {
731
- const section = totals.native;
732
- if (section !== null && file.native !== null) {
733
- const status = file.native.core.readPagedTotals(
734
- section,
735
- file.cache.pageCache(),
736
- first,
737
- output.length,
738
- output,
739
- fault
740
- );
741
- requireNativePage(status, file.path, "tokenTotals", fault[2]);
742
- return;
743
- }
744
- const bytes = totals.bytesAt(first * VALUE_BYTES, output.byteLength);
745
- const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
746
- for (let at = 0; at < output.length; at++) {
747
- const value = view.getFloat64(at * VALUE_BYTES, true);
748
- if (!Number.isFinite(value) || value < 0) loadFailed(`The term total at ${first + at} of "${file.path}" is invalid`);
749
- }
750
- for (let at = 0; at < output.length; at++) output[at] = view.getFloat64(at * VALUE_BYTES, true);
751
- }
752
-
753
1251
  // src/persistence/paged/expansion-source.ts
754
1252
  var NUMBER_BYTES = 8;
755
1253
  function createPagedExpansionSource(file, snapshot, table, tombstones, removedDocuments, removedOccurrences, totals) {
@@ -791,66 +1289,14 @@ function createPagedExpansionSource(file, snapshot, table, tombstones, removedDo
791
1289
  readPagedTotals(file, totals, 0, values, new BigUint64Array(3));
792
1290
  occurrences = values;
793
1291
  }
794
- return {
795
- growing: null,
796
- frozen,
797
- state,
798
- occurrences
799
- };
800
- },
801
- bytes: () => partialFields.byteLength + (occurrences?.byteLength ?? 0)
802
- };
803
- }
804
-
805
- // src/persistence/paged/removals.ts
806
- var VALUE_BYTES2 = 4;
807
- function createRemovalReader(file, removalOffsets, removals, tokenCount) {
808
- const fault = new BigUint64Array(3);
809
- return (ordinal) => {
810
- const bounds = removalOffsets.bytesAt(ordinal * VALUE_BYTES2, VALUE_BYTES2 * 2);
811
- const boundsView = new DataView(bounds.buffer, bounds.byteOffset, bounds.byteLength);
812
- const from = boundsView.getUint32(0, true);
813
- const to = boundsView.getUint32(VALUE_BYTES2, true);
814
- if (to < from || to * VALUE_BYTES2 > removals.length || from % 2 !== 0 || to % 2 !== 0 || ordinal === 0 && from !== 0 || (ordinal + 2) * VALUE_BYTES2 === removalOffsets.length && to * VALUE_BYTES2 !== removals.length) {
815
- loadFailed(`The removal entry of document ${ordinal} of "${file.path}" leaves its list`, { ordinal });
816
- }
817
- const values = new Uint32Array(to - from);
818
- const offsetSection = removalOffsets.native;
819
- const valueSection = removals.native;
820
- if (offsetSection !== null && valueSection !== null && file.native !== null) {
821
- const status = file.native.core.readPagedRemovals(
822
- offsetSection,
823
- valueSection,
824
- file.cache.pageCache(),
825
- ordinal,
826
- tokenCount,
827
- values,
828
- fault
829
- );
830
- if (status < 0)
831
- requireNativePage(
832
- status,
833
- file.path,
834
- fault[1] === offsetSection.offset ? "removalOffsets" : "removals",
835
- fault[2]
836
- );
837
- } else {
838
- const entry = removals.bytesAt(from * VALUE_BYTES2, values.byteLength);
839
- const view = new DataView(entry.buffer, entry.byteOffset, entry.byteLength);
840
- let previous = -1;
841
- for (let at = 0; at < values.length; at += 2) {
842
- const token = view.getUint32(at * VALUE_BYTES2, true);
843
- if (token >= tokenCount || token <= previous) {
844
- loadFailed(`The removal entry of document ${ordinal} of "${file.path}" holds invalid token ${token}`, {
845
- ordinal
846
- });
847
- }
848
- previous = token;
849
- values[at] = token;
850
- values[at + 1] = view.getUint32((at + 1) * VALUE_BYTES2, true);
851
- }
852
- }
853
- return values;
1292
+ return {
1293
+ growing: null,
1294
+ frozen,
1295
+ state,
1296
+ occurrences
1297
+ };
1298
+ },
1299
+ bytes: () => partialFields.byteLength + (occurrences?.byteLength ?? 0)
854
1300
  };
855
1301
  }
856
1302
 
@@ -979,375 +1425,117 @@ function createDocumentBlockReader(snapshot, documents, cache, fileNumber) {
979
1425
  { block }
980
1426
  );
981
1427
  }
982
- if (raw.byteLength !== rawLength) {
983
- loadFailed(`Document block ${block} of "${key}" inflates to ${raw.byteLength} bytes of ${rawLength}`, { block });
984
- }
985
- return raw;
986
- }
987
- function startsOf(block, raw) {
988
- const first = firstOrdinalOf(block);
989
- const end = block + 1 < blockCount ? firstOrdinalOf(block + 1) : documentCount;
990
- const starts = new Uint32Array(end - first + 1);
991
- const bytes = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
992
- let cursor = 0;
993
- for (let at = 0; at < end - first; at++) {
994
- starts[at] = cursor;
995
- const next = cursor < raw.byteLength ? endOfValue(bytes, cursor) : null;
996
- if (next === null)
997
- loadFailed(`Document block ${block} of "${key}" holds a broken document at ${at}`, { block, at });
998
- cursor = next;
999
- }
1000
- starts[end - first] = cursor;
1001
- if (cursor !== raw.byteLength)
1002
- loadFailed(`Document block ${block} of "${key}" holds bytes past its documents`, { block });
1003
- return starts;
1004
- }
1005
- function loadBlock(block) {
1006
- const held = cache.get(fileNumber, CACHED_DOCUMENT_BLOCK, block);
1007
- if (held !== void 0) return held;
1008
- const bytes = inflate(block);
1009
- const loaded = { bytes, starts: startsOf(block, bytes) };
1010
- cache.put(fileNumber, CACHED_DOCUMENT_BLOCK, block, loaded, bytes.byteLength + loaded.starts.byteLength);
1011
- return loaded;
1012
- }
1013
- return {
1014
- encodedDocumentAt(ordinal) {
1015
- if (ordinal < 0 || ordinal >= documentCount) {
1016
- loadFailed(`Document ${ordinal} lies outside "${key}", which holds ${documentCount}`, { ordinal });
1017
- }
1018
- const block = blockOf(ordinal);
1019
- const { bytes, starts } = loadBlock(block);
1020
- const row = ordinal - firstOrdinalOf(block);
1021
- return bytes.subarray(starts[row], starts[row + 1]);
1022
- }
1023
- };
1024
- }
1025
-
1026
- // src/persistence/paged/ids.ts
1027
- var VALUE_BYTES3 = 4;
1028
- var decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
1029
- function uint32At(reader, index) {
1030
- const bytes = reader.bytesAt(index * VALUE_BYTES3, VALUE_BYTES3);
1031
- return new DataView(bytes.buffer, bytes.byteOffset, VALUE_BYTES3).getUint32(0, true);
1032
- }
1033
- function createPagedIdTable(snapshot, ids, idOffsets, idOrder) {
1034
- const count = snapshot.documentCount;
1035
- const { idSampleIds, idSamplePositions, key } = snapshot;
1036
- let sampleBytes = idSamplePositions.byteLength;
1037
- for (const id of idSampleIds) sampleBytes += id.length;
1038
- function idAt(ordinal) {
1039
- const from = uint32At(idOffsets, ordinal);
1040
- const to = uint32At(idOffsets, ordinal + 1);
1041
- if (to < from || to > ids.length || ordinal === 0 && from !== 0 || ordinal + 1 === count && to !== ids.length)
1042
- loadFailed(`The ID of document ${ordinal} of "${key}" leaves its list`, { ordinal });
1043
- try {
1044
- return decoder.decode(ids.bytesAt(from, to - from));
1045
- } catch {
1046
- return loadFailed(`The ID of document ${ordinal} of "${key}" is not valid UTF-8`, { ordinal });
1047
- }
1048
- }
1049
- function ordinalAtPosition(position) {
1050
- const ordinal = uint32At(idOrder, position);
1051
- if (ordinal >= count) loadFailed(`idOrder of "${key}" names document ${ordinal} of ${count}`, { position });
1052
- return ordinal;
1053
- }
1054
- function windowOf(docId) {
1055
- let low = 0;
1056
- let high = idSampleIds.length;
1057
- while (low < high) {
1058
- const middle = low + high >>> 1;
1059
- if (compareCodePoints(idSampleIds[middle], docId) <= 0) low = middle + 1;
1060
- else high = middle;
1061
- }
1062
- const from = low === 0 ? 0 : idSamplePositions[low - 1];
1063
- const to = low < idSamplePositions.length ? idSamplePositions[low] + 1 : count;
1064
- return { from, to: Math.min(to, count) };
1065
- }
1066
- return {
1067
- count,
1068
- bytes: sampleBytes,
1069
- idAt,
1070
- ordinalOf(docId) {
1071
- if (count === 0) return -1;
1072
- const window = windowOf(docId);
1073
- let low = window.from;
1074
- let high = window.to;
1075
- while (low < high) {
1076
- const middle = low + high >>> 1;
1077
- const ordinal = ordinalAtPosition(middle);
1078
- const order = compareCodePoints(idAt(ordinal), docId);
1079
- if (order === 0) return ordinal;
1080
- if (order < 0) low = middle + 1;
1081
- else high = middle;
1082
- }
1083
- return -1;
1084
- },
1085
- sortedOrdinalAt: ordinalAtPosition,
1086
- collectSortedIds(excluded) {
1087
- const kept = [];
1088
- for (let position = 0; position < count; position++) {
1089
- const ordinal = ordinalAtPosition(position);
1090
- if (!excluded(ordinal)) kept.push(idAt(ordinal));
1091
- }
1092
- return kept;
1093
- }
1094
- };
1095
- }
1096
-
1097
- // src/persistence/paged/lists.ts
1098
- var LITTLE_ENDIAN_PLATFORM2 = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
1099
- function swappedInPlace(bytes, width) {
1100
- for (let at = 0; at + width <= bytes.length; at += width) {
1101
- for (let low = 0, high = width - 1; low < high; low++, high--) {
1102
- const held = bytes[at + low];
1103
- bytes[at + low] = bytes[at + high];
1104
- bytes[at + high] = held;
1105
- }
1106
- }
1107
- }
1108
- function fromLittleEndian(bytes, width) {
1109
- if (!LITTLE_ENDIAN_PLATFORM2) swappedInPlace(bytes, width);
1110
- }
1111
- function requireListRules(list, snapshot, token) {
1112
- const { docIds, fieldNameIndices } = list;
1113
- const documents = snapshot.documentCount;
1114
- const fields = snapshot.fieldNames.length;
1115
- for (let row = 0; row < docIds.length; row++) {
1116
- if (docIds[row] >= documents) {
1117
- loadFailed(`Token ${token} of "${snapshot.key}" names document ${docIds[row]} of ${documents}`, { token, row });
1118
- }
1119
- if (fieldNameIndices[row] >= fields) {
1120
- loadFailed(`Token ${token} of "${snapshot.key}" names field ${fieldNameIndices[row]} of ${fields}`, {
1121
- token,
1122
- row
1123
- });
1124
- }
1125
- if (row === 0) continue;
1126
- const before = docIds[row - 1];
1127
- if (docIds[row] < before || docIds[row] === before && fieldNameIndices[row] <= fieldNameIndices[row - 1]) {
1128
- loadFailed(`The list of token ${token} of "${snapshot.key}" descends at posting ${row}`, { token, row });
1129
- }
1130
- }
1131
- }
1132
- function createListReader(file, snapshot, sectionReader) {
1133
- const { postingOffsets } = snapshot;
1134
- const postings = sectionReader("postings");
1135
- const positionOffsets = sectionReader("positionOffsets");
1136
- const positions = sectionReader("positions");
1137
- const totals = sectionReader("tokenTotals");
1138
- const { cache, fileNumber } = file;
1139
- const fault = new BigUint64Array(3);
1140
- const total = new Float64Array(1);
1141
- function listAt(token) {
1142
- const held = cache.get(fileNumber, CACHED_LIST, token);
1143
- if (held !== void 0) return held;
1144
- if (postings === null) return loadFailed(`The segment "${snapshot.key}" holds no postings`);
1145
- const start = postingOffsets[token];
1146
- const count = postingOffsets[token + 1] - start;
1147
- const buffer = new ArrayBuffer(count * POSTING_WIDTH_BYTES);
1148
- const bytes = new Uint8Array(buffer);
1149
- const descriptor = postings.native;
1150
- if (descriptor !== null && file.native !== null) {
1151
- const status = file.native.core.readPagedPostings(
1152
- descriptor,
1153
- cache.pageCache(),
1154
- start,
1155
- count,
1156
- snapshot.documentCount,
1157
- snapshot.fieldNames.length,
1158
- bytes,
1159
- fault
1160
- );
1161
- requireNativePage(status, file.path, "postings", fault[2]);
1162
- } else {
1163
- postings.read(start * POSTING_WIDTH_BYTES, bytes);
1164
- }
1165
- if (!LITTLE_ENDIAN_PLATFORM2) {
1166
- swappedInPlace(bytes.subarray(0, count * 4), 4);
1167
- swappedInPlace(bytes.subarray(count * 4), 2);
1168
- }
1169
- const list = {
1170
- docIds: new Uint32Array(buffer, 0, count),
1171
- termFrequencies: new Uint16Array(buffer, count * 4, count),
1172
- fieldNameIndices: new Uint16Array(buffer, count * 6, count)
1173
- };
1174
- if (descriptor === null || file.native === null) requireListRules(list, snapshot, token);
1175
- cache.put(fileNumber, CACHED_LIST, token, list, buffer.byteLength);
1176
- return list;
1177
- }
1178
- function positionsAt(token) {
1179
- if (positionOffsets === null || positions === null) return null;
1180
- const held = cache.get(fileNumber, CACHED_POSITIONS, token);
1181
- if (held !== void 0) return held;
1182
- const start = postingOffsets[token];
1183
- const count = postingOffsets[token + 1] - start;
1184
- const offsetBytes = new Uint8Array((count + 1) * POSITION_WIDTH_BYTES);
1185
- positionOffsets.read(start * POSITION_WIDTH_BYTES, offsetBytes);
1186
- fromLittleEndian(offsetBytes, POSITION_WIDTH_BYTES);
1187
- const absolute = new Uint32Array(offsetBytes.buffer);
1188
- const first = absolute[0];
1189
- const end = absolute[count];
1190
- if (end < first || end * POSITION_WIDTH_BYTES > positions.length || start === 0 && first !== 0 || (start + count + 1) * POSITION_WIDTH_BYTES === positionOffsets.length && end * POSITION_WIDTH_BYTES !== positions.length)
1191
- loadFailed(`The positions of token ${token} of "${snapshot.key}" leave their section`, { token });
1192
- const offsets = new Uint32Array(count + 1);
1193
- const valueBytes = new Uint8Array((end - first) * POSITION_WIDTH_BYTES);
1194
- const loaded = { offsets, values: new Uint32Array(valueBytes.buffer) };
1195
- const offsetSection = positionOffsets.native;
1196
- const valueSection = positions.native;
1197
- if (offsetSection !== null && valueSection !== null && file.native !== null) {
1198
- const status = file.native.core.readPagedPositions(
1199
- offsetSection,
1200
- valueSection,
1201
- cache.pageCache(),
1202
- start,
1203
- count,
1204
- offsets,
1205
- loaded.values,
1206
- fault
1207
- );
1208
- requireNativePage(
1209
- status,
1210
- file.path,
1211
- fault[1] === offsetSection.offset ? "positionOffsets" : "positions",
1212
- fault[2]
1213
- );
1214
- } else {
1215
- for (let row = 0; row <= count; row++) {
1216
- if (absolute[row] < first || absolute[row] > end || row > 0 && absolute[row] < absolute[row - 1]) {
1217
- loadFailed(`The position offsets of token ${token} of "${snapshot.key}" leave their list`, { token, row });
1218
- }
1219
- offsets[row] = absolute[row] - first;
1220
- }
1221
- positions.read(first * POSITION_WIDTH_BYTES, valueBytes);
1222
- fromLittleEndian(valueBytes, POSITION_WIDTH_BYTES);
1223
- for (let row = 0; row < count; row++) {
1224
- for (let at = offsets[row] + 1; at < offsets[row + 1]; at++) {
1225
- if (loaded.values[at] < loaded.values[at - 1])
1226
- loadFailed(`The positions of token ${token} of "${snapshot.key}" descend`, { token, row });
1227
- }
1228
- }
1229
- }
1230
- cache.put(fileNumber, CACHED_POSITIONS, token, loaded, offsetBytes.byteLength + valueBytes.byteLength);
1428
+ if (raw.byteLength !== rawLength) {
1429
+ loadFailed(`Document block ${block} of "${key}" inflates to ${raw.byteLength} bytes of ${rawLength}`, { block });
1430
+ }
1431
+ return raw;
1432
+ }
1433
+ function startsOf(block, raw) {
1434
+ const first = firstOrdinalOf(block);
1435
+ const end = block + 1 < blockCount ? firstOrdinalOf(block + 1) : documentCount;
1436
+ const starts = new Uint32Array(end - first + 1);
1437
+ const bytes = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
1438
+ let cursor = 0;
1439
+ for (let at = 0; at < end - first; at++) {
1440
+ starts[at] = cursor;
1441
+ const next = cursor < raw.byteLength ? endOfValue(bytes, cursor) : null;
1442
+ if (next === null)
1443
+ loadFailed(`Document block ${block} of "${key}" holds a broken document at ${at}`, { block, at });
1444
+ cursor = next;
1445
+ }
1446
+ starts[end - first] = cursor;
1447
+ if (cursor !== raw.byteLength)
1448
+ loadFailed(`Document block ${block} of "${key}" holds bytes past its documents`, { block });
1449
+ return starts;
1450
+ }
1451
+ function loadBlock(block) {
1452
+ const held = cache.get(fileNumber, CACHED_DOCUMENT_BLOCK, block);
1453
+ if (held !== void 0) return held;
1454
+ const bytes = inflate(block);
1455
+ const loaded = { bytes, starts: startsOf(block, bytes) };
1456
+ cache.put(fileNumber, CACHED_DOCUMENT_BLOCK, block, loaded, bytes.byteLength + loaded.starts.byteLength);
1231
1457
  return loaded;
1232
1458
  }
1233
- function totalAt(token) {
1234
- if (totals === null) return loadFailed(`The segment "${snapshot.key}" holds no token totals`);
1235
- readPagedTotals(file, totals, token, total, fault);
1236
- return total[0];
1459
+ return {
1460
+ encodedDocumentAt(ordinal) {
1461
+ if (ordinal < 0 || ordinal >= documentCount) {
1462
+ loadFailed(`Document ${ordinal} lies outside "${key}", which holds ${documentCount}`, { ordinal });
1463
+ }
1464
+ const block = blockOf(ordinal);
1465
+ const { bytes, starts } = loadBlock(block);
1466
+ const row = ordinal - firstOrdinalOf(block);
1467
+ return bytes.subarray(starts[row], starts[row + 1]);
1468
+ }
1469
+ };
1470
+ }
1471
+
1472
+ // src/persistence/paged/ids.ts
1473
+ var VALUE_BYTES3 = 4;
1474
+ var decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
1475
+ function uint32At(reader, index) {
1476
+ const bytes = reader.bytesAt(index * VALUE_BYTES3, VALUE_BYTES3);
1477
+ return new DataView(bytes.buffer, bytes.byteOffset, VALUE_BYTES3).getUint32(0, true);
1478
+ }
1479
+ function createPagedIdTable(snapshot, ids, idOffsets, idOrder) {
1480
+ const count = snapshot.documentCount;
1481
+ const { idSampleIds, idSamplePositions, key } = snapshot;
1482
+ let sampleBytes = idSamplePositions.byteLength;
1483
+ for (const id of idSampleIds) sampleBytes += id.length;
1484
+ function idAt(ordinal) {
1485
+ const from = uint32At(idOffsets, ordinal);
1486
+ const to = uint32At(idOffsets, ordinal + 1);
1487
+ if (to < from || to > ids.length || ordinal === 0 && from !== 0 || ordinal + 1 === count && to !== ids.length)
1488
+ loadFailed(`The ID of document ${ordinal} of "${key}" leaves its list`, { ordinal });
1489
+ try {
1490
+ return decoder.decode(ids.bytesAt(from, to - from));
1491
+ } catch {
1492
+ return loadFailed(`The ID of document ${ordinal} of "${key}" is not valid UTF-8`, { ordinal });
1493
+ }
1237
1494
  }
1238
- function listedIndexOf(token) {
1239
- const { blockLists } = snapshot;
1495
+ function ordinalAtPosition(position) {
1496
+ const ordinal = uint32At(idOrder, position);
1497
+ if (ordinal >= count) loadFailed(`idOrder of "${key}" names document ${ordinal} of ${count}`, { position });
1498
+ return ordinal;
1499
+ }
1500
+ function windowOf(docId) {
1240
1501
  let low = 0;
1241
- let high = blockLists.length;
1502
+ let high = idSampleIds.length;
1242
1503
  while (low < high) {
1243
1504
  const middle = low + high >>> 1;
1244
- if (blockLists[middle] < token) low = middle + 1;
1505
+ if (compareCodePoints(idSampleIds[middle], docId) <= 0) low = middle + 1;
1245
1506
  else high = middle;
1246
1507
  }
1247
- return low < blockLists.length && blockLists[low] === token ? low : -1;
1248
- }
1249
- function storedBoundsAt(token) {
1250
- const listed = listedIndexOf(token);
1251
- if (listed < 0) return null;
1252
- const first = snapshot.blockListOffsets[listed];
1253
- const blockCount = snapshot.blockListOffsets[listed + 1] - first;
1254
- const { blockSummaries } = snapshot;
1255
- const view = new DataView(blockSummaries.buffer, blockSummaries.byteOffset, blockSummaries.byteLength);
1256
- const bounds = {
1257
- structureRevision: 0,
1258
- blockCount,
1259
- entryEnd: new Int32Array(blockCount),
1260
- maxTermFrequency: new Int32Array(blockCount),
1261
- minFieldLength: new Float64Array(blockCount),
1262
- maxEntriesPerDocument: new Int32Array(blockCount),
1263
- documentCount: new Int32Array(blockCount)
1264
- };
1265
- const length = postingOffsets[token + 1] - postingOffsets[token];
1266
- for (let block = 0; block < blockCount; block++) {
1267
- const place = (first + block) * BLOCK_SUMMARY_BYTES;
1268
- bounds.entryEnd[block] = view.getUint32(place, true);
1269
- bounds.maxTermFrequency[block] = view.getUint16(place + 4, true);
1270
- bounds.minFieldLength[block] = view.getFloat64(place + 6, true);
1271
- bounds.maxEntriesPerDocument[block] = view.getUint32(place + 14, true);
1272
- bounds.documentCount[block] = view.getUint32(place + 18, true);
1273
- const before = block === 0 ? 0 : bounds.entryEnd[block - 1];
1274
- if (bounds.entryEnd[block] <= before || bounds.entryEnd[block] > length) {
1275
- loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" ends outside its list`, { token, block });
1276
- }
1277
- }
1278
- if (blockCount === 0 || bounds.entryEnd[blockCount - 1] !== length) {
1279
- loadFailed(`The blocks of token ${token} of "${snapshot.key}" fail to cover its list`, { token });
1280
- }
1281
- return bounds;
1282
- }
1283
- return { listAt, positionsAt, totalAt, storedBoundsAt };
1284
- }
1285
- function createPagedPostingViews(snapshot, lists, tombstones, counts) {
1286
- const { postingOffsets, tracksPositions } = snapshot;
1287
- const views = /* @__PURE__ */ new Map();
1288
- function buildView(token) {
1289
- const length = postingOffsets[token + 1] - postingOffsets[token];
1290
- return {
1291
- length,
1292
- get docIds() {
1293
- return lists.listAt(token).docIds;
1294
- },
1295
- get termFrequencies() {
1296
- return lists.listAt(token).termFrequencies;
1297
- },
1298
- get fieldNameIndices() {
1299
- return lists.listAt(token).fieldNameIndices;
1300
- },
1301
- get positions() {
1302
- const loaded = lists.positionsAt(token);
1303
- if (loaded === null) return null;
1304
- const rows = new Array(length);
1305
- for (let row = 0; row < length; row++) {
1306
- rows[row] = [...loaded.values.subarray(loaded.offsets[row], loaded.offsets[row + 1])];
1307
- }
1308
- return rows;
1309
- },
1310
- holdsPositions: tracksPositions,
1311
- positionCountAt(row) {
1312
- const loaded = lists.positionsAt(token);
1313
- return loaded === null ? 0 : loaded.offsets[row + 1] - loaded.offsets[row];
1314
- },
1315
- get positionStore() {
1316
- return lists.positionsAt(token)?.values ?? null;
1317
- },
1318
- positionStartAt(row) {
1319
- return lists.positionsAt(token)?.offsets[row] ?? 0;
1320
- },
1321
- docIdSet: {
1322
- get size() {
1323
- return counts.documentFrequency(token);
1324
- }
1325
- },
1326
- deletedDocs: tombstones,
1327
- get totalTermFrequency() {
1328
- return lists.totalAt(token) - counts.removedOccurrences(token);
1329
- },
1330
- get structureRevision() {
1331
- return tombstones.revision;
1332
- },
1333
- ordered: true,
1334
- storedBlockBounds() {
1335
- return tombstones.size === 0 ? lists.storedBoundsAt(token) : null;
1336
- }
1337
- };
1508
+ const from = low === 0 ? 0 : idSamplePositions[low - 1];
1509
+ const to = low < idSamplePositions.length ? idSamplePositions[low] + 1 : count;
1510
+ return { from, to: Math.min(to, count) };
1338
1511
  }
1339
1512
  return {
1340
- viewAt(payloadSlot) {
1341
- let view = views.get(payloadSlot);
1342
- if (view === void 0) {
1343
- if (views.size >= LIST_VIEWS_HELD_PER_SEGMENT) views.clear();
1344
- view = buildView(payloadSlot);
1345
- views.set(payloadSlot, view);
1513
+ count,
1514
+ bytes: sampleBytes,
1515
+ idAt,
1516
+ ordinalOf(docId) {
1517
+ if (count === 0) return -1;
1518
+ const window = windowOf(docId);
1519
+ let low = window.from;
1520
+ let high = window.to;
1521
+ while (low < high) {
1522
+ const middle = low + high >>> 1;
1523
+ const ordinal = ordinalAtPosition(middle);
1524
+ const order = compareCodePoints(idAt(ordinal), docId);
1525
+ if (order === 0) return ordinal;
1526
+ if (order < 0) low = middle + 1;
1527
+ else high = middle;
1346
1528
  }
1347
- return view;
1529
+ return -1;
1348
1530
  },
1349
- viewReadOnce(payloadSlot) {
1350
- return buildView(payloadSlot);
1531
+ sortedOrdinalAt: ordinalAtPosition,
1532
+ collectSortedIds(excluded) {
1533
+ const kept = [];
1534
+ for (let position = 0; position < count; position++) {
1535
+ const ordinal = ordinalAtPosition(position);
1536
+ if (!excluded(ordinal)) kept.push(idAt(ordinal));
1537
+ }
1538
+ return kept;
1351
1539
  }
1352
1540
  };
1353
1541
  }
@@ -1577,76 +1765,59 @@ function assembleOpenedSegment(file, snapshot, cache) {
1577
1765
  });
1578
1766
  }
1579
1767
 
1580
- // src/persistence/paged/verify.ts
1581
- async function orLoadFailure(key, read) {
1582
- try {
1583
- return await read();
1584
- } catch (error) {
1585
- if (error instanceof NarsilError) throw error;
1586
- return loadFailed(
1587
- `The segment "${key}" cannot be read: ${error instanceof Error ? error.message : String(error)}`,
1588
- {
1589
- key
1590
- }
1591
- );
1592
- }
1593
- }
1594
- async function envelopeVersionOf(directory, key) {
1595
- const { readRange } = directory;
1596
- if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
1597
- const bytes = await orLoadFailure(key, () => readRange.call(directory, key, 4, 1));
1598
- return bytes[0];
1599
- }
1600
- async function isPagedSegmentFile(directory, key) {
1601
- return await envelopeVersionOf(directory, key) === PAGED_SEGMENT_ENVELOPE_VERSION;
1602
- }
1603
- async function requirePagedPayloadChecksum(directory, listing) {
1604
- const { readRange } = directory;
1605
- const { key, bytes } = listing;
1606
- if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
1607
- const header2 = readHeader(await readRange.call(directory, key, 0, HEADER_SIZE));
1608
- if (HEADER_SIZE + header2.payloadLength !== bytes) {
1609
- loadFailed(`The segment "${key}" holds ${bytes} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`);
1610
- }
1611
- await requirePayloadChecksum(key, header2, (offset, length) => readRange.call(directory, key, offset, length));
1612
- }
1613
- async function requirePagedPathChecksum(path) {
1614
- const read = async (offset, length) => {
1615
- const bytes = new Uint8Array(length);
1616
- if (!await readFileRangeLater(path, offset, bytes))
1617
- loadFailed(`The segment "${path}" cannot be read at ${offset}`);
1618
- return bytes;
1768
+ // src/persistence/paged/segment-scan.ts
1769
+ async function openPagedSegmentScan(directory, ref) {
1770
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
1771
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
1772
+ const segment = createPagedFrozenSegment(snapshot, cache);
1773
+ const { paged } = segment;
1774
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
1775
+ const ids = segment.arrays.idTable;
1776
+ const tombstones = new Set(snapshot.tombstones);
1777
+ return {
1778
+ count: snapshot.documentCount,
1779
+ schema: snapshot.schema,
1780
+ tombstones,
1781
+ idAt: (ordinal) => ids.idAt(ordinal),
1782
+ holds: (docId) => ids.ordinalOf(docId) >= 0,
1783
+ storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
1784
+ close: () => {
1785
+ paged.close();
1786
+ cache.clear();
1787
+ }
1619
1788
  };
1620
- await requirePayloadChecksum(path, readHeader(await read(0, HEADER_SIZE)), read);
1621
1789
  }
1622
- async function requirePayloadChecksum(key, header2, read) {
1623
- const bytes = HEADER_SIZE + header2.payloadLength;
1624
- let state = crc32Init();
1625
- for (let at = HEADER_SIZE; at < bytes; at += FILE_ASSEMBLY_CHUNK_BYTES) {
1626
- const size = Math.min(FILE_ASSEMBLY_CHUNK_BYTES, bytes - at);
1627
- state = crc32Update(state, await read(at, size));
1790
+ async function checkPagedSegmentFully(directory, ref) {
1791
+ const listing = await pagedListingOf(directory, ref);
1792
+ await requirePagedPayloadChecksum(directory, listing);
1793
+ const snapshot = openPagedSnapshot(listing);
1794
+ const cache = createReadCache(MERGE_READ_CACHE_BYTES);
1795
+ const segment = createPagedFrozenSegment(snapshot, cache);
1796
+ const { paged } = segment;
1797
+ if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
1798
+ try {
1799
+ await checkPagedContents(segment, cache);
1800
+ return snapshot.schema;
1801
+ } finally {
1802
+ paged.close();
1803
+ cache.clear();
1628
1804
  }
1629
- const computed = crc32Final(state);
1630
- if (computed !== header2.checksum) {
1631
- throw new NarsilError(
1632
- ErrorCodes.PERSISTENCE_CRC_MISMATCH,
1633
- `The checksum of the segment "${key}" does not match its payload`,
1634
- { key, expected: header2.checksum, computed }
1635
- );
1805
+ }
1806
+ async function bytesThatSegmentsHoldInMemory(directory, manifest) {
1807
+ let bytes = 0;
1808
+ for (const partition of manifest.partitions) {
1809
+ for (const ref of partition.segments) {
1810
+ const paged = await isPagedSegmentFile(directory, ref.key);
1811
+ bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
1812
+ }
1636
1813
  }
1814
+ return bytes;
1637
1815
  }
1638
- async function pagedListingOf(directory, ref) {
1639
- const { byteLength } = directory;
1640
- const length = byteLength === void 0 ? ref.bytes : await byteLength.call(directory, ref.key);
1641
- if (length === null) loadFailed(`The segment "${ref.key}" that the manifest lists is missing`, { key: ref.key });
1642
- if (length !== ref.bytes) {
1643
- loadFailed(`The segment "${ref.key}" holds ${length} bytes where the manifest lists ${ref.bytes}`, {
1644
- key: ref.key,
1645
- bytes: length,
1646
- listed: ref.bytes
1647
- });
1816
+ function countsAsLatestHolder(scans, at, docId) {
1817
+ for (let later = at + 1; later < scans.length; later++) {
1818
+ if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
1648
1819
  }
1649
- return { key: ref.key, path: await directory.pathOf(ref.key), bytes: ref.bytes, checksum: ref.checksum };
1820
+ return true;
1650
1821
  }
1651
1822
 
1652
1823
  // src/persistence/paged/older-holders.ts
@@ -1702,122 +1873,6 @@ async function pagedTombstonesFor(directory, older, written, deleted) {
1702
1873
  return [...tombstones];
1703
1874
  }
1704
1875
 
1705
- // src/persistence/paged/check-contents.ts
1706
- function requireIds(segment, snapshot) {
1707
- const ids = segment.arrays.idTable;
1708
- let previous = null;
1709
- for (let position = 0; position < snapshot.documentCount; position++) {
1710
- const id = ids.idAt(ids.sortedOrdinalAt(position));
1711
- if (previous !== null && compareCodePoints(previous, id) >= 0)
1712
- loadFailed(`The ID order of "${snapshot.key}" does not ascend at position ${position}`, { position });
1713
- if (position % ID_SAMPLE_INTERVAL === 0 && snapshot.idSampleIds[position / ID_SAMPLE_INTERVAL] !== id)
1714
- loadFailed(`The ID sample of "${snapshot.key}" differs at position ${position}`, { position });
1715
- previous = id;
1716
- }
1717
- }
1718
- function requirePostings(lists, snapshot) {
1719
- const fields = snapshot.fieldNames.map((field) => {
1720
- const at = snapshot.fieldLengthNames.indexOf(field);
1721
- return at < 0 ? null : snapshot.fieldLengthColumns[at];
1722
- });
1723
- for (let token = 0; token < snapshot.tokenTable.documentFrequencies.length; token++) {
1724
- const list = lists.listAt(token);
1725
- lists.positionsAt(token);
1726
- const bounds = lists.storedBoundsAt(token);
1727
- if (list.docIds.length > SMALLEST_BLOCK_POSTINGS !== (bounds !== null))
1728
- loadFailed(`The block summaries of token ${token} of "${snapshot.key}" do not cover its postings`, { token });
1729
- let documents = 0;
1730
- let occurrences = 0;
1731
- let block = 0;
1732
- let start = 0;
1733
- let maxFrequency = 0;
1734
- let minLength = Number.POSITIVE_INFINITY;
1735
- let maxEntries = 0;
1736
- let blockDocuments = 0;
1737
- let row = 0;
1738
- while (row < list.docIds.length) {
1739
- const document = list.docIds[row];
1740
- let entries = 0;
1741
- do {
1742
- const frequency = list.termFrequencies[row];
1743
- occurrences += frequency;
1744
- maxFrequency = Math.max(maxFrequency, frequency);
1745
- minLength = Math.min(minLength, fields[list.fieldNameIndices[row]]?.[document] ?? 0);
1746
- row++;
1747
- entries++;
1748
- } while (row < list.docIds.length && list.docIds[row] === document);
1749
- documents++;
1750
- blockDocuments++;
1751
- maxEntries = Math.max(maxEntries, entries);
1752
- if (bounds === null || row - start < SMALLEST_BLOCK_POSTINGS && row < list.docIds.length) continue;
1753
- if (block >= bounds.blockCount || bounds.entryEnd[block] !== row || bounds.maxTermFrequency[block] !== maxFrequency || bounds.minFieldLength[block] !== minLength || bounds.maxEntriesPerDocument[block] !== maxEntries || bounds.documentCount[block] !== blockDocuments)
1754
- loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" differs from its postings`, { token, block });
1755
- block++;
1756
- start = row;
1757
- maxFrequency = 0;
1758
- minLength = Number.POSITIVE_INFINITY;
1759
- maxEntries = 0;
1760
- blockDocuments = 0;
1761
- }
1762
- if (documents !== snapshot.tokenTable.documentFrequencies[token] || occurrences !== lists.totalAt(token))
1763
- loadFailed(`The counts of token ${token} of "${snapshot.key}" differ from its postings`, { token });
1764
- }
1765
- }
1766
- async function checkPagedContents(segment, cache) {
1767
- const { paged } = segment;
1768
- if (paged === null) return;
1769
- const { snapshot } = paged;
1770
- for (const section of snapshot.sections) await checkEverySectionPage(snapshot.path, section, () => true);
1771
- const file = openPagedFile(snapshot.path, cache);
1772
- try {
1773
- const sectionReader = (name) => {
1774
- const section = sectionNamed(snapshot.sections, name);
1775
- return section === void 0 ? null : createSectionReader(file, section);
1776
- };
1777
- const required = (name) => {
1778
- const section = sectionNamed(snapshot.sections, name);
1779
- if (section === void 0) loadFailed(`The segment "${snapshot.key}" holds no section "${name}"`);
1780
- return createSectionReader(file, section);
1781
- };
1782
- const removals = createRemovalReader(
1783
- file,
1784
- required("removalOffsets"),
1785
- required("removals"),
1786
- segment.arrays.tokenTable.size
1787
- );
1788
- for (let ordinal = 0; ordinal < snapshot.documentCount; ordinal++) {
1789
- decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields);
1790
- paged.writeRecordAt(ordinal);
1791
- removals(ordinal);
1792
- }
1793
- if (snapshot.documentCount === 0) {
1794
- for (const name of ["idOffsets", "removalOffsets"]) {
1795
- const offsets = required(name).bytesAt(0, 4);
1796
- if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
1797
- loadFailed(`Section "${name}" of "${snapshot.key}" does not start at zero`);
1798
- }
1799
- }
1800
- if (snapshot.tracksPositions && snapshot.postingOffsets[snapshot.postingOffsets.length - 1] === 0) {
1801
- const offsets = required("positionOffsets").bytesAt(0, 4);
1802
- if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
1803
- loadFailed(`The position offsets of "${snapshot.key}" do not start at zero`);
1804
- }
1805
- requireIds(segment, snapshot);
1806
- requirePostings(createListReader(file, snapshot, sectionReader), snapshot);
1807
- segment.arrays.readSurfaceForms();
1808
- } finally {
1809
- closePagedFile(file);
1810
- }
1811
- }
1812
- async function checkPagedSelections(selections) {
1813
- for (const { segment } of selections) {
1814
- const { paged } = segment;
1815
- if (paged === null || paged.payloadChecked) continue;
1816
- await requirePagedPathChecksum(paged.snapshot.path);
1817
- paged.payloadChecked = true;
1818
- }
1819
- }
1820
-
1821
1876
  // src/core/segment-merges/metadata.ts
1822
1877
  function emptySource(input) {
1823
1878
  return {
@@ -3608,61 +3663,6 @@ async function mergePagedWindow(input) {
3608
3663
  }
3609
3664
  }
3610
3665
 
3611
- // src/persistence/paged/segment-scan.ts
3612
- async function openPagedSegmentScan(directory, ref) {
3613
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
3614
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
3615
- const segment = createPagedFrozenSegment(snapshot, cache);
3616
- const { paged } = segment;
3617
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
3618
- const ids = segment.arrays.idTable;
3619
- const tombstones = new Set(snapshot.tombstones);
3620
- return {
3621
- count: snapshot.documentCount,
3622
- schema: snapshot.schema,
3623
- tombstones,
3624
- idAt: (ordinal) => ids.idAt(ordinal),
3625
- holds: (docId) => ids.ordinalOf(docId) >= 0,
3626
- storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
3627
- close: () => {
3628
- paged.close();
3629
- cache.clear();
3630
- }
3631
- };
3632
- }
3633
- async function checkPagedSegmentFully(directory, ref) {
3634
- const listing = await pagedListingOf(directory, ref);
3635
- await requirePagedPayloadChecksum(directory, listing);
3636
- const snapshot = openPagedSnapshot(listing);
3637
- const cache = createReadCache(MERGE_READ_CACHE_BYTES);
3638
- const segment = createPagedFrozenSegment(snapshot, cache);
3639
- const { paged } = segment;
3640
- if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
3641
- try {
3642
- await checkPagedContents(segment, cache);
3643
- return snapshot.schema;
3644
- } finally {
3645
- paged.close();
3646
- cache.clear();
3647
- }
3648
- }
3649
- async function bytesThatSegmentsHoldInMemory(directory, manifest) {
3650
- let bytes = 0;
3651
- for (const partition of manifest.partitions) {
3652
- for (const ref of partition.segments) {
3653
- const paged = await isPagedSegmentFile(directory, ref.key);
3654
- bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
3655
- }
3656
- }
3657
- return bytes;
3658
- }
3659
- function countsAsLatestHolder(scans, at, docId) {
3660
- for (let later = at + 1; later < scans.length; later++) {
3661
- if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
3662
- }
3663
- return true;
3664
- }
3665
-
3666
3666
  // src/persistence/durability/partition-checkpoint.ts
3667
3667
  function checkpointLastSeqNo(checkpoint, partitionId) {
3668
3668
  for (const c of checkpoint) {
@@ -3905,6 +3905,9 @@ function validateEntryPayload(decoded) {
3905
3905
  if (entry.document !== null && !(entry.document instanceof Uint8Array)) {
3906
3906
  throw new Error('Invalid EntryPayload: "entry.document" must be Uint8Array or null');
3907
3907
  }
3908
+ if ("previousDocument" in entry && entry.previousDocument !== null && !(entry.previousDocument instanceof Uint8Array)) {
3909
+ throw new Error('Invalid EntryPayload: "entry.previousDocument" must be Uint8Array or null where present');
3910
+ }
3908
3911
  if (typeof entry.checksum !== "number") {
3909
3912
  throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
3910
3913
  }
@@ -4045,9 +4048,15 @@ function validateAckPayload(decoded) {
4045
4048
  return decoded;
4046
4049
  }
4047
4050
 
4051
+ // src/distribution/replication/constants.ts
4052
+ var REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES = 40;
4053
+ var DEFAULT_LOG_RETENTION_BYTES = 268435456;
4054
+ var INSYNC_CAS_ATTEMPTS = 5;
4055
+ var SNAPSHOT_CHUNK_SIZE = 65536;
4056
+
4048
4057
  // src/distribution/replication/entry-checksum.ts
4049
- function computeEntryChecksum(entry) {
4050
- const payload = encode([
4058
+ function checksumFields(entry) {
4059
+ const fields = [
4051
4060
  entry.seqNo,
4052
4061
  entry.primaryTerm,
4053
4062
  entry.operation,
@@ -4055,24 +4064,54 @@ function computeEntryChecksum(entry) {
4055
4064
  entry.indexName,
4056
4065
  entry.documentId,
4057
4066
  entry.document
4058
- ]);
4059
- return crc32(payload);
4067
+ ];
4068
+ if (entry.previousDocument !== void 0) fields.push(entry.previousDocument);
4069
+ return fields;
4070
+ }
4071
+ function computeEntryChecksum(entry) {
4072
+ return crc32(encode(checksumFields(entry)));
4060
4073
  }
4061
4074
  function buildEntry(fields) {
4062
- return { ...fields, checksum: computeEntryChecksum(fields) };
4075
+ if (fields.previousDocument === void 0) {
4076
+ return {
4077
+ seqNo: fields.seqNo,
4078
+ primaryTerm: fields.primaryTerm,
4079
+ operation: fields.operation,
4080
+ partitionId: fields.partitionId,
4081
+ indexName: fields.indexName,
4082
+ documentId: fields.documentId,
4083
+ document: fields.document,
4084
+ checksum: computeEntryChecksum(fields)
4085
+ };
4086
+ }
4087
+ return {
4088
+ seqNo: fields.seqNo,
4089
+ primaryTerm: fields.primaryTerm,
4090
+ operation: fields.operation,
4091
+ partitionId: fields.partitionId,
4092
+ indexName: fields.indexName,
4093
+ documentId: fields.documentId,
4094
+ document: fields.document,
4095
+ previousDocument: fields.previousDocument,
4096
+ checksum: computeEntryChecksum(fields)
4097
+ };
4098
+ }
4099
+ function entryByteEstimate(entry) {
4100
+ return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0) + (entry.previousDocument?.byteLength ?? 0);
4063
4101
  }
4064
4102
 
4065
4103
  // src/persistence/durability/wal-framing.ts
4066
4104
  var WAL_MAGIC = new Uint8Array([78, 82, 83, 87]);
4067
4105
  var WAL_FORMAT_VERSION = 1;
4106
+ var WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT = 2;
4068
4107
  var SEGMENT_HEADER_SIZE = 8;
4069
4108
  var RECORD_LENGTH_SIZE = 4;
4070
4109
  var FRAME_CRC_SIZE = 4;
4071
4110
  var MAX_WAL_RECORD_BYTES = 268435456;
4072
- function writeSegmentHeader() {
4111
+ function writeSegmentHeader(formatVersion = WAL_FORMAT_VERSION) {
4073
4112
  const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
4074
4113
  header2.set(WAL_MAGIC, 0);
4075
- header2[4] = WAL_FORMAT_VERSION & 255;
4114
+ header2[4] = formatVersion & 255;
4076
4115
  return header2;
4077
4116
  }
4078
4117
  function frameRecord(entry) {
@@ -4094,10 +4133,10 @@ function checkSegmentHeader(data) {
4094
4133
  }
4095
4134
  }
4096
4135
  const formatVersion = data[4];
4097
- if (formatVersion > WAL_FORMAT_VERSION) {
4136
+ if (formatVersion > WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT) {
4098
4137
  return {
4099
4138
  ok: false,
4100
- reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}, so a newer release wrote this segment`,
4139
+ reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT}, so a newer release wrote this segment`,
4101
4140
  code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
4102
4141
  };
4103
4142
  }
@@ -4215,12 +4254,38 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
4215
4254
  }
4216
4255
 
4217
4256
  // src/persistence/durability/wal-segments.ts
4257
+ var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
4218
4258
  function walPrefix(indexName, partitionId) {
4219
4259
  return `${indexName}/wal/${partitionId}/`;
4220
4260
  }
4261
+ function segmentKey2(indexName, partitionId, startSeqNo) {
4262
+ return `${walPrefix(indexName, partitionId)}${startSeqNo.toString().padStart(16, "0")}`;
4263
+ }
4264
+ function parseSegmentStartSeqNo(key) {
4265
+ const tail = key.slice(key.lastIndexOf("/") + 1);
4266
+ if (!SEGMENT_TAIL_PATTERN.test(tail)) {
4267
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Malformed WAL segment key: "${key}"`, { key });
4268
+ }
4269
+ const value = Number.parseInt(tail, 10);
4270
+ if (!Number.isSafeInteger(value)) {
4271
+ throw new NarsilError(
4272
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
4273
+ `WAL segment start sequence number is out of range: "${key}"`,
4274
+ { key }
4275
+ );
4276
+ }
4277
+ return value;
4278
+ }
4221
4279
  function nextSegmentKey(indexName, partitionId) {
4222
4280
  return `${walPrefix(indexName, partitionId)}next`;
4223
4281
  }
4282
+ function rewrittenHeaderKey(indexName, partitionId) {
4283
+ return `${walPrefix(indexName, partitionId)}header`;
4284
+ }
4285
+ async function removeUnusedSegmentFiles(directory, indexName, partitionId) {
4286
+ await directory.remove(nextSegmentKey(indexName, partitionId));
4287
+ await directory.remove(rewrittenHeaderKey(indexName, partitionId));
4288
+ }
4224
4289
  function segmentStartSeqNo(key, prefix) {
4225
4290
  if (!key.startsWith(prefix)) {
4226
4291
  return null;
@@ -4924,4 +4989,4 @@ async function removeCheckpointGarbage(store, garbage) {
4924
4989
  }
4925
4990
  }
4926
4991
 
4927
- export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, residentSectionsOf, savedBytesOfManifest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
4992
+ export { DEFAULT_LOG_RETENTION_BYTES, FRAME_CRC_SIZE, INSYNC_CAS_ATTEMPTS, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, NOTHING_SAVED, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, SEGMENT_TAIL_PATTERN, SNAPSHOT_CHUNK_SIZE, WAL_FORMAT_VERSION, WAL_FORMAT_VERSION_WITH_PREVIOUS_DOCUMENT, buildEntry, bytesThatSegmentsHoldInMemory, captureWholePartition, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkSegmentHeader, checkpointGarbage, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, countsAsLatestHolder, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, entryByteEstimate, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSegmentScan, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, parseSegmentStartSeqNo, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, removeUnusedSegmentFiles, residentSectionsOf, rewrittenHeaderKey, savedBytesOfManifest, segmentKey2 as segmentKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };