@delali/narsil 0.3.1-next.185 → 0.3.1-next.189

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +1 -0
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  7. package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
  8. package/dist/browser/{chunk-CPSHS46N.mjs → chunk-WFQLSNCO.mjs} +27 -22
  9. package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
  10. package/dist/browser/{module-XVLHJ7T7.mjs → module-VOTN3Q6X.mjs} +15 -0
  11. package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
  12. package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-62LA56U2.mjs} +14558 -13739
  13. package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
  14. package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
  15. package/dist/{chunk-7WXGUTWK.mjs → chunk-3JRN23HA.mjs} +17 -6
  16. package/dist/{chunk-5BUEBIXR.mjs → chunk-3OZ2UQJV.mjs} +18 -5
  17. package/dist/{chunk-VYSWVQCE.mjs → chunk-5HPILB7R.mjs} +2 -2
  18. package/dist/{chunk-FBQYC5OX.mjs → chunk-6G3VRWH5.mjs} +16 -13
  19. package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
  20. package/dist/{chunk-H4QYQ5DE.mjs → chunk-7NCVV3ST.mjs} +2 -2
  21. package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
  22. package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
  23. package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
  24. package/dist/{chunk-DU23J5JG.mjs → chunk-EFS5GVBK.mjs} +6 -6
  25. package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
  26. package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
  27. package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
  28. package/dist/{chunk-LR6JJVML.mjs → chunk-GYLRKKUT.mjs} +764 -699
  29. package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-IDWDYS2N.mjs} +1 -1
  30. package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
  31. package/dist/{chunk-BVZWWMOU.mjs → chunk-IWKYAO45.mjs} +2 -2
  32. package/dist/chunk-JEMK23OY.mjs +602 -0
  33. package/dist/{chunk-2YWWHMVC.mjs → chunk-JOWXHENJ.mjs} +2 -2
  34. package/dist/{chunk-SL53DHVE.mjs → chunk-JXDY3ZA3.mjs} +66 -40
  35. package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
  36. package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
  37. package/dist/{chunk-2KCCQK4R.mjs → chunk-NFAWR2O4.mjs} +2 -2
  38. package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
  39. package/dist/{chunk-MHBG33R5.mjs → chunk-NM2ITITI.mjs} +76 -46
  40. package/dist/{chunk-WSIU6YXQ.mjs → chunk-NULDX3ID.mjs} +1732 -743
  41. package/dist/{chunk-II6XF6M4.mjs → chunk-P2IPUYT2.mjs} +6 -6
  42. package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
  43. package/dist/{chunk-A3OAFUZI.mjs → chunk-PLLKT5NU.mjs} +2 -2
  44. package/dist/{chunk-FCQMZDRI.mjs → chunk-PPD4J2JI.mjs} +3 -3
  45. package/dist/{chunk-EDEE7ETQ.mjs → chunk-PZYBWMCL.mjs} +3 -3
  46. package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
  47. package/dist/{chunk-M5I64AQW.mjs → chunk-REBDKXAH.mjs} +33 -33
  48. package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
  49. package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
  50. package/dist/{chunk-O337MDRJ.mjs → chunk-U5IMECEX.mjs} +38 -13
  51. package/dist/{chunk-CODGHJVW.mjs → chunk-X2NE4VJI.mjs} +40 -12
  52. package/dist/{chunk-2GONI5HL.mjs → chunk-YFMM7QOT.mjs} +5 -4
  53. package/dist/{chunk-JDG6CCXM.mjs → chunk-YQ4RZIIN.mjs} +2 -2
  54. package/dist/{chunk-G3XRHAJA.mjs → chunk-ZGMCJXY5.mjs} +3 -3
  55. package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
  56. package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
  57. package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
  58. package/dist/client.d.ts +7 -7
  59. package/dist/client.mjs +3 -1265
  60. package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
  61. package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
  62. package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
  63. package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-QLBSVTEE.mjs} +8 -8
  64. package/dist/distribution/coordinator/etcd.d.ts +1 -1
  65. package/dist/distribution/coordinator/etcd.mjs +35 -15
  66. package/dist/distribution/coordinator/in-memory.d.ts +1 -1
  67. package/dist/distribution/coordinator/in-memory.mjs +16 -6
  68. package/dist/distribution/transport/grpc.mjs +2 -2
  69. package/dist/distribution/transport/tcp.mjs +2 -2
  70. package/dist/distribution.d.ts +70 -9
  71. package/dist/distribution.mjs +1694 -679
  72. package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
  73. package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
  74. package/dist/embeddings/openai.mjs +2 -2
  75. package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
  76. package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
  77. package/dist/index.browser.mjs +3 -3
  78. package/dist/index.d.ts +4 -4
  79. package/dist/index.mjs +8 -8
  80. package/dist/index.react-native.mjs +9 -9
  81. package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-IKD4Z37M.mjs} +21 -21
  82. package/dist/{module-Z3CL4GJ4.mjs → module-NIVZ4SF2.mjs} +22 -10
  83. package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
  84. package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
  85. package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-DVTVX2XY.mjs} +31 -18
  86. package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-ES4Z5SM4.mjs} +20 -20
  87. package/dist/partial-words.browser.mjs +3 -3
  88. package/dist/partial-words.mjs +8 -8
  89. package/dist/pattern.browser.mjs +2 -2
  90. package/dist/pattern.mjs +3 -3
  91. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
  92. package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
  93. package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
  94. package/dist/{query-actions-7EOKT3WD.mjs → query-actions-IC5UEWZG.mjs} +8 -8
  95. package/dist/query-suggestions/merge-worker.mjs +11 -11
  96. package/dist/query-suggestions.browser.mjs +3 -3
  97. package/dist/query-suggestions.d.ts +1 -1
  98. package/dist/query-suggestions.mjs +9 -9
  99. package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-3553RYCS.mjs} +3 -3
  100. package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
  101. package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
  103. package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-FJHZWUCB.mjs} +5 -5
  104. package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
  105. package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-GXY4R3GA.mjs} +2 -2
  106. package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
  107. package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-JZOHEHBD.mjs} +3 -3
  108. package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
  109. package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
  110. package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
  111. package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-VSN76LWE.mjs} +1 -1
  112. package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-XLYQ2ZMC.mjs} +222 -47
  113. package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
  114. package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
  115. package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
  116. package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
  117. package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
  118. package/dist/react-native/chunks/{module-COLNJNML.mjs → module-TN7HXWJ3.mjs} +21 -6
  119. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  120. package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
  121. package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-2UPNQNAN.mjs} +31 -18
  122. package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
  123. package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
  124. package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-DEAWTVHW.mjs} +12 -12
  125. package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-3LZ6HZFD.mjs} +10 -10
  126. package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-5OIWAPCI.mjs} +11 -11
  127. package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-LC4RZU3T.mjs} +15614 -14805
  128. package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-LQ6LWM2Q.mjs} +11 -11
  129. package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-GPNU6I72.mjs} +11 -11
  130. package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-32TCHOYP.mjs} +10 -10
  131. package/dist/react-native/partial-words.mjs +9 -9
  132. package/dist/react-native/pattern.mjs +5 -5
  133. package/dist/react-native/query-suggestions.mjs +9 -9
  134. package/dist/react-native/sync.mjs +433 -51
  135. package/dist/react-native/synonyms.mjs +9 -9
  136. package/dist/react.d.ts +8 -8
  137. package/dist/react.mjs +7 -0
  138. package/dist/{rerank-KTV5L7MZ.mjs → rerank-ELMGHQ3T.mjs} +8 -8
  139. package/dist/rerankers/http.mjs +2 -2
  140. package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
  141. package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-6VCVZT5C.mjs} +11 -11
  142. package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-XNMCUIJF.mjs} +9 -9
  143. package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-RA4S7CC6.mjs} +9 -9
  144. package/dist/server.d.ts +13 -9
  145. package/dist/server.mjs +875 -348
  146. package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-36SAEVV3.mjs} +5243 -5846
  147. package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-QAGHHLEK.mjs} +10 -10
  148. package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-JG2IP2VQ.mjs} +12 -12
  149. package/dist/sync/filesystem.d.ts +2 -2
  150. package/dist/sync/filesystem.mjs +16 -16
  151. package/dist/sync/indexeddb.browser.mjs +1 -1
  152. package/dist/sync/indexeddb.d.ts +2 -2
  153. package/dist/sync/indexeddb.mjs +3 -3
  154. package/dist/sync.browser.mjs +35 -1108
  155. package/dist/sync.d.ts +3 -3
  156. package/dist/sync.mjs +430 -51
  157. package/dist/synonyms.browser.mjs +3 -3
  158. package/dist/synonyms.mjs +8 -8
  159. package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
  160. package/dist/thread-HAE2XXFR.mjs +31 -0
  161. package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
  162. package/dist/vector/search-worker.mjs +5 -5
  163. package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-IXC43Z4Q.mjs} +11 -11
  164. package/dist/workers/entry.mjs +14 -14
  165. package/openapi.json +144 -0
  166. package/package.json +7 -7
  167. package/dist/chunk-URVH6BLO.mjs +0 -217
  168. package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
  169. package/dist/thread-NQRNW3IH.mjs +0 -31
  170. /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
@@ -1,25 +1,167 @@
1
- import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-II6XF6M4.mjs';
2
- import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-URVH6BLO.mjs';
3
- import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-7WXGUTWK.mjs';
4
- import { writeCheckpointSegments, commitCheckpointManifest, isPagedSegmentFile, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE } from './chunk-LR6JJVML.mjs';
5
- import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-M5I64AQW.mjs';
6
- import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-2KCCQK4R.mjs';
1
+ import { loadVectorField, openColdVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-P2IPUYT2.mjs';
2
+ import { engineChangeSource, ChangeCursor, createSemaphore, changePositionList, changesNotEnabled, previousDocumentRequired, invalidPositions, changePositionsOf, requireConfigurationShape, RESTORE_ESTIMATE_MARGIN, persistEpochStates, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-JEMK23OY.mjs';
3
+ import { restoredConfigFields, restoredEmbedding } from './chunk-M2Z5WVWW.mjs';
4
+ import { ENTRIES_MESSAGE_LIMIT_BYTES, FEED_HEAD_RETRY_MS, decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-3JRN23HA.mjs';
5
+ import { isPagedSegmentFile, openPagedSegmentScan, countsAsLatestHolder, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, removeUnusedSegmentFiles, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-GYLRKKUT.mjs';
6
+ import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-REBDKXAH.mjs';
7
+ import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-NFAWR2O4.mjs';
7
8
  import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-MSIN4NM3.mjs';
8
- import { encode } from './chunk-CV3Q2EKO.mjs';
9
- import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
10
- import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-4JBTALUC.mjs';
11
- import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-WLFOLAQI.mjs';
12
- import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-EDEE7ETQ.mjs';
13
- import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-SL53DHVE.mjs';
9
+ import { encode } from './chunk-S5QMWECL.mjs';
10
+ import { setNestedValue, restoreVectorFields } from './chunk-R2PJKSNL.mjs';
11
+ import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-IDWDYS2N.mjs';
12
+ import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-KBU3ZFJN.mjs';
13
+ import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
14
+ import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-PZYBWMCL.mjs';
15
+ import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, noteDocumentChurn, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, withOpenIndex, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, decodeStoredDocument, emitSyncChange, ARRAY_POSITION_GAP } from './chunk-JXDY3ZA3.mjs';
14
16
  import { decode } from './chunk-GXCDWLQI.mjs';
15
- import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
16
- import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-X5NHWOXT.mjs';
17
- import { collectAndReturnPages, turnIsSpent, yieldTurn, countWrite } from './chunk-UPDNNOTG.mjs';
18
- import { DEFAULT_MAX_PARTITIONS, ANALYSIS_CHANGE_MEMBERS, MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-3ECIRFS5.mjs';
17
+ import { isSyncEpoch, newSyncEpoch } from './chunk-BUMREPLU.mjs';
18
+ import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, parseVectorDimension, requireValidFieldTypes, isQuantizationMode, isStorageMode, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-7P5YWDRQ.mjs';
19
+ import { collectAndReturnPages, turnsAreBudgeted, turnIsSpent, yieldTurn, countWrite } from './chunk-UPDNNOTG.mjs';
20
+ import { DEFAULT_MAX_PARTITIONS, ANALYSIS_CHANGE_MEMBERS, EXPORT_CHUNK_BYTES, MAX_PARTITIONS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-3ECIRFS5.mjs';
19
21
  import { VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
20
- import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
22
+ import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
21
23
  import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
22
24
 
25
+ // src/changes/constants.ts
26
+ var CHANGES_IDLE_RECHECK_MS = 1e3;
27
+ var MAX_FOLLOWER_CATCH_UPS = 4;
28
+
29
+ // src/changes/follow.ts
30
+ function requestOf(indexName, options) {
31
+ const { epoch, positions, filter } = options;
32
+ if (epoch === void 0 !== (positions === void 0)) {
33
+ throw invalidPositions(indexName, "Pass epoch and positions together, or leave both out");
34
+ }
35
+ const request = filter === void 0 ? {} : { filter };
36
+ if (epoch === void 0) return request;
37
+ if (!isSyncEpoch(epoch)) throw invalidPositions(indexName, "The epoch must be 32 lowercase hexadecimal digits");
38
+ const valid = changePositionsOf(positions);
39
+ if (valid === null) {
40
+ throw invalidPositions(indexName, "Each position needs a whole partitionId and a whole, non-negative seqNo");
41
+ }
42
+ return { ...request, epoch, positions: valid };
43
+ }
44
+ function documentOf(bytes, vectorFieldPaths) {
45
+ const document = decode(bytes);
46
+ restoreVectorFields(document, vectorFieldPaths);
47
+ return document;
48
+ }
49
+ function eventOf(batch, vectorFieldPaths) {
50
+ const changes = batch.changes.map((change) => {
51
+ const shaped = { seqNo: change.seqNo, type: change.type, documentId: change.documentId };
52
+ if (change.document !== null) shaped.document = documentOf(change.document, vectorFieldPaths);
53
+ if (change.previousDocument !== null) {
54
+ shaped.previousDocument = documentOf(change.previousDocument, vectorFieldPaths);
55
+ }
56
+ return shaped;
57
+ });
58
+ return { type: "changes", partitionId: batch.partitionId, changes, scannedSeqNo: batch.scannedSeqNo };
59
+ }
60
+ function wakeupOn(source, indexName, signal) {
61
+ let pending = false;
62
+ let wake = null;
63
+ const fire = () => {
64
+ pending = true;
65
+ wake?.();
66
+ };
67
+ const stopListening = source.listen((change) => {
68
+ if (change.indexName === indexName) fire();
69
+ });
70
+ const stopWatching = source.watchIndex?.(indexName);
71
+ signal?.addEventListener("abort", fire);
72
+ return {
73
+ async wait(ms) {
74
+ if (pending) {
75
+ pending = false;
76
+ return;
77
+ }
78
+ let timer;
79
+ await new Promise((resolve) => {
80
+ wake = resolve;
81
+ timer = setTimeout(resolve, ms);
82
+ });
83
+ clearTimeout(timer);
84
+ wake = null;
85
+ pending = false;
86
+ },
87
+ stop() {
88
+ stopListening();
89
+ stopWatching?.();
90
+ signal?.removeEventListener("abort", fire);
91
+ }
92
+ };
93
+ }
94
+ function followIndexChanges(core, indexName, options = {}) {
95
+ core.guardShutdown();
96
+ core.requireIndex(indexName);
97
+ return followSourceChanges(
98
+ { source: engineChangeSource(core), guard: () => core.guardShutdown() },
99
+ indexName,
100
+ options
101
+ );
102
+ }
103
+ async function* followSourceChanges(deps, indexName, options = {}) {
104
+ deps.guard();
105
+ const request = requestOf(indexName, options);
106
+ const { source } = deps;
107
+ const { signal } = options;
108
+ const aborted = () => signal?.aborted === true;
109
+ if (aborted()) return;
110
+ const wakeup = wakeupOn(source, indexName, signal);
111
+ try {
112
+ const opened = await ChangeCursor.open(
113
+ { source, indexName, reads: createSemaphore(MAX_FOLLOWER_CATCH_UPS) },
114
+ request
115
+ );
116
+ if (typeof opened === "string") {
117
+ yield { type: "resync", reason: opened };
118
+ return;
119
+ }
120
+ const cursor = opened;
121
+ const { state } = cursor;
122
+ const vectorFieldPaths = await source.vectorFieldPaths(indexName);
123
+ yield {
124
+ type: "subscribed",
125
+ epoch: state.epoch,
126
+ partitionCount: state.partitionCount,
127
+ positions: changePositionList(cursor.subscribedSeqNos),
128
+ heads: changePositionList(cursor.heads),
129
+ bytesBehind: cursor.bytesBehind,
130
+ previousDocument: state.previousDocument
131
+ };
132
+ while (!aborted()) {
133
+ deps.guard();
134
+ const current = source.changesState(indexName);
135
+ if (current === null) throw changesNotEnabled(indexName);
136
+ if (current.epoch !== state.epoch) {
137
+ yield { type: "resync", reason: "epoch" };
138
+ return;
139
+ }
140
+ if (cursor.filtered && !current.previousDocument) throw previousDocumentRequired(indexName);
141
+ await cursor.refreshHeads();
142
+ let progressed = false;
143
+ for (let partitionId = 0; partitionId < state.partitionCount; partitionId++) {
144
+ if (aborted() || !cursor.behind(partitionId)) continue;
145
+ const batch = await cursor.readBatch(partitionId, ENTRIES_MESSAGE_LIMIT_BYTES);
146
+ if (batch === "history") {
147
+ yield { type: "resync", reason: "history" };
148
+ return;
149
+ }
150
+ if (batch === null) continue;
151
+ const afterRead = source.changesState(indexName);
152
+ if (afterRead === null || afterRead.epoch !== state.epoch) break;
153
+ if (cursor.filtered && !afterRead.previousDocument) throw previousDocumentRequired(indexName);
154
+ cursor.commit(batch);
155
+ progressed = true;
156
+ yield eventOf(batch, vectorFieldPaths);
157
+ }
158
+ if (!progressed) await wakeup.wait(cursor.awaitsDurability() ? FEED_HEAD_RETRY_MS : CHANGES_IDLE_RECHECK_MS);
159
+ }
160
+ } finally {
161
+ wakeup.stop();
162
+ }
163
+ }
164
+
23
165
  // src/engine/analysis-change/pending-target.ts
24
166
  function configUnderAnalysis(base, language, tokenizer, stopWords) {
25
167
  const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
@@ -1004,11 +1146,11 @@ function requireRestoreMemory(indexName, footprint, replacesIndex) {
1004
1146
  // src/engine/analysis-change/run.ts
1005
1147
  function createHold(generation) {
1006
1148
  let markReady = () => void 0;
1007
- let refuse2 = () => void 0;
1149
+ let refuse5 = () => void 0;
1008
1150
  let wake = () => void 0;
1009
1151
  const ready = new Promise((resolve, reject) => {
1010
1152
  markReady = resolve;
1011
- refuse2 = reject;
1153
+ refuse5 = reject;
1012
1154
  });
1013
1155
  ready.catch(() => void 0);
1014
1156
  return {
@@ -1017,7 +1159,7 @@ function createHold(generation) {
1017
1159
  released: false,
1018
1160
  ready,
1019
1161
  markReady,
1020
- refuse: refuse2,
1162
+ refuse: refuse5,
1021
1163
  wake: () => wake(),
1022
1164
  pause: () => new Promise((resolve) => {
1023
1165
  const timer = setTimeout(resolve, ANALYSIS_HOLD_APPLY_INTERVAL_MS);
@@ -1483,85 +1625,6 @@ async function stampAnalysisGeneration(core, indexName, generation) {
1483
1625
  await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
1484
1626
  }
1485
1627
 
1486
- // src/partitioning/saved-partitions.ts
1487
- function requireSavedPartitionCount(partitionCount, source, indexName) {
1488
- if (partitionCount <= MAX_PARTITIONS) return;
1489
- throw new NarsilError(
1490
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1491
- `The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
1492
- { indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
1493
- );
1494
- }
1495
- function highestPartitionIdOf(partitions) {
1496
- let highest = partitions.length - 1;
1497
- for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
1498
- return highest;
1499
- }
1500
-
1501
- // src/persistence/paged/read-ahead.ts
1502
- var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
1503
- function rangeOf(path, section) {
1504
- return { path, offset: HEADER_SIZE + section.offset, length: section.length };
1505
- }
1506
- async function readRangesAhead(ranges) {
1507
- let rangeAt = 0;
1508
- let doneInRange = 0;
1509
- function nextChunk() {
1510
- while (rangeAt < ranges.length && doneInRange >= ranges[rangeAt].length) {
1511
- rangeAt += 1;
1512
- doneInRange = 0;
1513
- }
1514
- if (rangeAt >= ranges.length) return null;
1515
- const { path, offset, length } = ranges[rangeAt];
1516
- const chunk = { path, offset: offset + doneInRange, length: Math.min(READ_AHEAD_CHUNK_BYTES, length - doneInRange) };
1517
- doneInRange += chunk.length;
1518
- return chunk;
1519
- }
1520
- const scratch = new Uint8Array(READ_AHEAD_CHUNK_BYTES);
1521
- async function readChunks() {
1522
- for (let chunk = nextChunk(); chunk !== null; chunk = nextChunk()) {
1523
- const target = scratch.subarray(0, chunk.length);
1524
- if (!await readFileRangeLater(chunk.path, chunk.offset, target)) return;
1525
- }
1526
- }
1527
- await Promise.all(Array.from({ length: READS_AHEAD_AT_ONCE }, readChunks));
1528
- }
1529
- async function headOf(path, ref) {
1530
- const { headOffset, headLength } = ref;
1531
- if (headOffset === void 0 || headLength === void 0) return null;
1532
- const head = new Uint8Array(headLength);
1533
- if (!await readFileRangeLater(path, HEADER_SIZE + headOffset, head)) return null;
1534
- return decode(head);
1535
- }
1536
- async function levelsOfPart(directory, ref) {
1537
- const path = await directory.pathOf(ref.key);
1538
- const head = await headOf(path, ref);
1539
- const graphs = head?.graphs;
1540
- if (!Array.isArray(graphs) || graphs.length !== 1) return [];
1541
- const { levels } = graphs[0];
1542
- return decodeSections([levels], ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES).map(
1543
- (section) => rangeOf(path, section)
1544
- );
1545
- }
1546
- async function residentSectionsOfSegment(directory, ref) {
1547
- if (!await isPagedSegmentFile(directory, ref.key)) return [];
1548
- const listing = await pagedListingOf(directory, ref);
1549
- return residentSectionsOf(listing).map((section) => rangeOf(listing.path, section));
1550
- }
1551
- async function startReadingOpeningSectionsAhead(directory, manifest) {
1552
- try {
1553
- const groups = await Promise.all([
1554
- ...manifest.partitions.flatMap(
1555
- (partition) => partition.segments.map((ref) => residentSectionsOfSegment(directory, ref))
1556
- ),
1557
- ...manifest.vectors.flatMap((field) => field.graphParts.map((ref) => levelsOfPart(directory, ref)))
1558
- ]);
1559
- void readRangesAhead(groups.flat());
1560
- } catch {
1561
- return;
1562
- }
1563
- }
1564
-
1565
1628
  // src/persistence/archive/byte-reader.ts
1566
1629
  async function* chunksOf(source) {
1567
1630
  if (source instanceof Uint8Array) {
@@ -1639,327 +1702,1017 @@ function createByteReader(source) {
1639
1702
  };
1640
1703
  }
1641
1704
 
1642
- // src/engine/durability-metadata.ts
1643
- var VECTOR_TYPE = /^vector\[(\d+)\]$/;
1644
- function vectorFieldsOf(schema, promotion) {
1645
- const fields = {};
1646
- for (const [fieldPath, type] of Object.entries(schema)) {
1647
- const match = VECTOR_TYPE.exec(type);
1648
- if (match === null) continue;
1649
- const dimension = Number(match[1]);
1650
- fields[fieldPath] = {
1651
- dimension,
1652
- metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
1653
- quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
1654
- };
1705
+ // src/engine/snapshot/stored-schema.ts
1706
+ function isRecord2(value) {
1707
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1708
+ }
1709
+ function conflictingStoredPath(path) {
1710
+ return new NarsilError(
1711
+ ErrorCodes.SCHEMA_INVALID_TYPE,
1712
+ `Field path "${path}" is declared both as a field and as an object of fields`,
1713
+ { path }
1714
+ );
1715
+ }
1716
+ function defineOwn(level, key, value) {
1717
+ Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
1718
+ }
1719
+ function placeStoredField(nested, path, type) {
1720
+ const segments = path.split(".");
1721
+ let level = nested;
1722
+ for (let at = 0; at < segments.length; at++) {
1723
+ const segment = segments[at];
1724
+ const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
1725
+ if (at === segments.length - 1) {
1726
+ if (held !== void 0) throw conflictingStoredPath(path);
1727
+ defineOwn(level, segment, type);
1728
+ return;
1729
+ }
1730
+ if (held === void 0) {
1731
+ const child = {};
1732
+ defineOwn(level, segment, child);
1733
+ level = child;
1734
+ } else if (isRecord2(held)) {
1735
+ level = held;
1736
+ } else {
1737
+ throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
1738
+ }
1655
1739
  }
1656
- return Object.keys(fields).length === 0 ? null : fields;
1657
1740
  }
1658
- function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
1659
- const metadata = {
1660
- indexName,
1661
- schema: config.schema,
1662
- language: config.language,
1663
- partitionCount: partitionCount ?? config.partitionCount,
1664
- bm25Params: {
1665
- k1: config.k1,
1666
- b: config.b,
1667
- ...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
1668
- },
1669
- createdAt: Date.now(),
1670
- engineVersion: VERSION
1671
- };
1672
- metadata.documentCount = documentCount ?? config.documentCount;
1673
- const recordedSavedBytes = savedBytes ?? config.savedBytes;
1674
- if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
1675
- const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
1676
- if (vectorFields !== null) {
1677
- metadata.vectorFields = vectorFields;
1741
+ function placeStoredFields(nested, stored, prefix) {
1742
+ for (const [key, type] of Object.entries(stored)) {
1743
+ const path = prefix ? `${prefix}.${key}` : key;
1744
+ if (isRecord2(type)) {
1745
+ placeStoredFields(nested, type, path);
1746
+ } else {
1747
+ placeStoredField(nested, path, type);
1748
+ }
1678
1749
  }
1679
- if (config.embedding) {
1680
- metadata.embedding = config.embedding;
1750
+ }
1751
+ function nestedSchemaOf(stored) {
1752
+ if (!isRecord2(stored)) return stored;
1753
+ const nested = {};
1754
+ placeStoredFields(nested, stored, "");
1755
+ return nested;
1756
+ }
1757
+
1758
+ // src/engine/export/lines.ts
1759
+ var NEWLINE = 10;
1760
+ function joined(parts, total) {
1761
+ if (parts.length === 1) return parts[0];
1762
+ const bytes = new Uint8Array(total);
1763
+ let offset = 0;
1764
+ for (const part of parts) {
1765
+ bytes.set(part, offset);
1766
+ offset += part.byteLength;
1681
1767
  }
1682
- if (config.surfaceForms !== void 0) {
1683
- metadata.surfaceForms = config.surfaceForms;
1768
+ return bytes;
1769
+ }
1770
+ async function* exportLines(source, maxLineBytes) {
1771
+ const decoder2 = new TextDecoder("utf-8", { fatal: true });
1772
+ let parts = [];
1773
+ let pending = 0;
1774
+ let overflowing = false;
1775
+ let line = 0;
1776
+ const finishLine = () => {
1777
+ line += 1;
1778
+ if (overflowing) {
1779
+ overflowing = false;
1780
+ parts = [];
1781
+ pending = 0;
1782
+ return { line, refused: "too-long" };
1783
+ }
1784
+ const bytes = joined(parts, pending);
1785
+ parts = [];
1786
+ pending = 0;
1787
+ try {
1788
+ return { line, text: decoder2.decode(bytes) };
1789
+ } catch {
1790
+ return { line, refused: "not-utf8" };
1791
+ }
1792
+ };
1793
+ for await (const chunk of chunksOf(source)) {
1794
+ let start = 0;
1795
+ for (; ; ) {
1796
+ const newline = chunk.indexOf(NEWLINE, start);
1797
+ const end = newline < 0 ? chunk.byteLength : newline;
1798
+ const piece = chunk.subarray(start, end);
1799
+ if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
1800
+ if (piece.byteLength > 0) parts.push(piece.slice());
1801
+ pending += piece.byteLength;
1802
+ } else {
1803
+ overflowing = true;
1804
+ parts = [];
1805
+ }
1806
+ if (newline < 0) break;
1807
+ yield finishLine();
1808
+ start = newline + 1;
1809
+ }
1684
1810
  }
1685
- if (config.analysisRevision !== void 0) {
1686
- metadata.analysisRevision = config.analysisRevision;
1811
+ if (pending > 0 || overflowing) yield finishLine();
1812
+ }
1813
+
1814
+ // src/serialization/base64.ts
1815
+ var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
1816
+ var PAD = 61;
1817
+ var BYTES_PER_GROUP = 3;
1818
+ var CHARACTERS_PER_GROUP = 4;
1819
+ var ASCII_LIMIT = 128;
1820
+ var ENCODE = new Uint8Array(ALPHABET.length);
1821
+ var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
1822
+ for (let at = 0; at < ALPHABET.length; at++) {
1823
+ ENCODE[at] = ALPHABET.charCodeAt(at);
1824
+ DECODE[ALPHABET.charCodeAt(at)] = at;
1825
+ }
1826
+ var asciiDecoder = new TextDecoder();
1827
+ function encodeBase64(bytes) {
1828
+ const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
1829
+ let write = 0;
1830
+ let read = 0;
1831
+ for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
1832
+ const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
1833
+ out[write++] = ENCODE[group >>> 18];
1834
+ out[write++] = ENCODE[group >>> 12 & 63];
1835
+ out[write++] = ENCODE[group >>> 6 & 63];
1836
+ out[write++] = ENCODE[group & 63];
1837
+ }
1838
+ const left = bytes.byteLength - read;
1839
+ if (left > 0) {
1840
+ const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
1841
+ out[write++] = ENCODE[group >>> 18];
1842
+ out[write++] = ENCODE[group >>> 12 & 63];
1843
+ out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
1844
+ out[write++] = PAD;
1845
+ }
1846
+ return asciiDecoder.decode(out);
1847
+ }
1848
+ function sextetAt(text, at) {
1849
+ const code = text.charCodeAt(at);
1850
+ return code < ASCII_LIMIT ? DECODE[code] : -1;
1851
+ }
1852
+ function decodeExactBase64(text) {
1853
+ if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
1854
+ const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
1855
+ const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
1856
+ const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
1857
+ let write = 0;
1858
+ for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
1859
+ const a2 = sextetAt(text, at);
1860
+ const b2 = sextetAt(text, at + 1);
1861
+ const c2 = sextetAt(text, at + 2);
1862
+ const d = sextetAt(text, at + 3);
1863
+ if ((a2 | b2 | c2 | d) < 0) return null;
1864
+ const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
1865
+ out[write++] = group >>> 16;
1866
+ out[write++] = group >>> 8 & 255;
1867
+ out[write++] = group & 255;
1868
+ }
1869
+ if (padding === 0) return out;
1870
+ const a = sextetAt(text, fullEnd);
1871
+ const b = sextetAt(text, fullEnd + 1);
1872
+ const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
1873
+ if ((a | b | c) < 0) return null;
1874
+ if (padding === 2 && (b & 15) !== 0) return null;
1875
+ if (padding === 1 && (c & 3) !== 0) return null;
1876
+ out[write++] = a << 2 | b >>> 4;
1877
+ if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
1878
+ return out;
1879
+ }
1880
+
1881
+ // src/engine/export/tags.ts
1882
+ var TAG_PREFIX = "$";
1883
+ var DATE_TAG = "$date";
1884
+ var BYTES_TAG = "$bytes";
1885
+ var NUMBER_TAG = "$number";
1886
+ var OBJECT_TAG = "$object";
1887
+ var MILLISECONDS_PER_SECOND = 1e3;
1888
+ var NANOSECONDS_PER_MILLISECOND = 1e6;
1889
+ var MAX_NANOSECONDS = 999999999;
1890
+ var MAX_DATE_MILLISECONDS = 864e13;
1891
+ var ESCAPED_TAG_PREFIX = "\\u0024";
1892
+ var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
1893
+ ["NaN", Number.NaN],
1894
+ ["Infinity", Number.POSITIVE_INFINITY],
1895
+ ["-Infinity", Number.NEGATIVE_INFINITY]
1896
+ ]);
1897
+ function refuse2(path, reason) {
1898
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
1899
+ field: path
1900
+ });
1901
+ }
1902
+ function isRecord3(value) {
1903
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1904
+ }
1905
+ function startsWithTagPrefix(key) {
1906
+ return key.startsWith(TAG_PREFIX);
1907
+ }
1908
+ function soleTagKey(record) {
1909
+ let sole = null;
1910
+ for (const key of Object.keys(record)) {
1911
+ if (sole !== null) return null;
1912
+ sole = key;
1913
+ }
1914
+ return sole !== null && startsWithTagPrefix(sole) ? sole : null;
1915
+ }
1916
+ function dateOf(value, path) {
1917
+ if (!Array.isArray(value) || value.length !== 2)
1918
+ refuse2(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
1919
+ const seconds = value[0];
1920
+ const nanoseconds = value[1];
1921
+ if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
1922
+ refuse2(
1923
+ path,
1924
+ `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
1925
+ );
1687
1926
  }
1688
- if (config.arrayPositionGap !== void 0) {
1689
- metadata.arrayPositionGap = config.arrayPositionGap;
1927
+ if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
1928
+ refuse2(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
1690
1929
  }
1691
- if (config.tokenizer !== void 0) {
1692
- metadata.tokenizer = config.tokenizer;
1930
+ const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
1931
+ if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
1932
+ refuse2(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
1933
+ return new Date(milliseconds);
1934
+ }
1935
+ function bytesOf(value, path) {
1936
+ const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
1937
+ if (bytes === null) refuse2(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
1938
+ return bytes;
1939
+ }
1940
+ function numberOf(value, path) {
1941
+ const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
1942
+ if (number === void 0) refuse2(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
1943
+ return number;
1944
+ }
1945
+ function requireDepth(path, depth) {
1946
+ if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse2(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
1947
+ }
1948
+ function untagMap(record, path, depth) {
1949
+ requireDepth(path, depth);
1950
+ for (const key of Object.keys(record)) {
1951
+ record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
1952
+ }
1953
+ return record;
1954
+ }
1955
+ function untagList(list, path, depth) {
1956
+ requireDepth(path, depth);
1957
+ for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
1958
+ return list;
1959
+ }
1960
+ function untagTag(tag, value, path, depth) {
1961
+ switch (tag) {
1962
+ case DATE_TAG:
1963
+ return dateOf(value, path);
1964
+ case BYTES_TAG:
1965
+ return bytesOf(value, path);
1966
+ case NUMBER_TAG:
1967
+ return numberOf(value, path);
1968
+ case OBJECT_TAG:
1969
+ if (!isRecord3(value)) refuse2(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
1970
+ return untagMap(value, path, depth);
1971
+ default:
1972
+ return refuse2(path, `holds an object whose only key, "${tag}", names no tag`);
1973
+ }
1974
+ }
1975
+ function untag(value, path, depth) {
1976
+ if (typeof value !== "object" || value === null) return value;
1977
+ if (Array.isArray(value)) return untagList(value, path, depth);
1978
+ const record = value;
1979
+ const tag = soleTagKey(record);
1980
+ return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
1981
+ }
1982
+ function untagDocumentLine(text, parsed) {
1983
+ if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
1984
+ return untagMap(parsed, "", 0);
1985
+ }
1986
+
1987
+ // src/engine/export/import.ts
1988
+ var DOCUMENT_EXPORT_FORMAT = "narsil-export";
1989
+ var DOCUMENT_EXPORT_VERSION = 1;
1990
+ function invalidHeader(message) {
1991
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
1992
+ }
1993
+ function configOfHeader(line) {
1994
+ if (line === void 0) invalidHeader("is missing");
1995
+ if (!("text" in line)) {
1996
+ invalidHeader(
1997
+ line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
1998
+ );
1693
1999
  }
1694
- if (config.stopWords !== void 0) {
1695
- metadata.stopWords = config.stopWords;
2000
+ let raw;
2001
+ try {
2002
+ raw = JSON.parse(line.text);
2003
+ } catch {
2004
+ invalidHeader("is not JSON");
1696
2005
  }
1697
- if (config.stopWordList !== void 0) {
1698
- metadata.stopWordList = config.stopWordList;
2006
+ if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
2007
+ invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
1699
2008
  }
1700
- if (config.analysisGeneration !== void 0) {
1701
- metadata.analysisGeneration = config.analysisGeneration;
2009
+ const configuration = requireConfigurationShape(raw.index, "document export header");
2010
+ const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
2011
+ const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
2012
+ return {
2013
+ schema: nestedSchemaOf(configuration.schema),
2014
+ language: configuration.language,
2015
+ ...fields,
2016
+ ...embedding === void 0 ? {} : {
2017
+ embedding: {
2018
+ fields: embedding.fields,
2019
+ ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
2020
+ }
2021
+ }
2022
+ };
2023
+ }
2024
+ async function prepareIndex(engine, indexName, config) {
2025
+ if (!engine.listIndexes().some((index) => index.name === indexName)) {
2026
+ await engine.createIndex(indexName, config);
2027
+ return;
1702
2028
  }
1703
- if (config.pendingAnalysis !== void 0) {
1704
- metadata.pendingAnalysis = config.pendingAnalysis;
2029
+ const existing = flattenSchema(engine.getStats(indexName).schema);
2030
+ const exported = flattenSchema(config.schema);
2031
+ const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
2032
+ for (const field of fields) {
2033
+ if (existing[field] === exported[field]) continue;
2034
+ throw new NarsilError(
2035
+ ErrorCodes.CONFIG_INVALID,
2036
+ `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
2037
+ { indexName, field }
2038
+ );
1705
2039
  }
1706
- if (config.partitionLimits !== void 0) {
1707
- metadata.partitionLimits = config.partitionLimits;
2040
+ }
2041
+ function highestReportedAt(refused) {
2042
+ let highest = 0;
2043
+ for (let at = 1; at < refused.length; at++) {
2044
+ if (refused[at].line > refused[highest].line) highest = at;
1708
2045
  }
1709
- if (config.defaultScoring !== void 0) {
1710
- metadata.defaultScoring = config.defaultScoring;
2046
+ return highest;
2047
+ }
2048
+ function refuse3(state, line, code, message) {
2049
+ state.result.refusedCount += 1;
2050
+ const refused = state.result.refused;
2051
+ if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
2052
+ refused.push({ line, code, message });
2053
+ if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
2054
+ return;
1711
2055
  }
1712
- if (config.trackPositions !== void 0) {
1713
- metadata.trackPositions = config.trackPositions;
1714
- }
1715
- if (config.textStorage !== void 0) {
1716
- metadata.textStorage = config.textStorage;
1717
- }
1718
- if (config.strict !== void 0) {
1719
- metadata.strict = config.strict;
2056
+ if (line >= refused[state.highestReported].line) return;
2057
+ refused[state.highestReported] = { line, code, message };
2058
+ state.highestReported = highestReportedAt(refused);
2059
+ }
2060
+ async function flush(state) {
2061
+ if (state.batch.length === 0) return;
2062
+ const batch = state.batch;
2063
+ state.batch = [];
2064
+ state.batchIds = /* @__PURE__ */ new Set();
2065
+ const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
2066
+ const inserts = [];
2067
+ const updates = [];
2068
+ for (const entry of batch) {
2069
+ if (await state.engine.has(state.indexName, entry.docId))
2070
+ updates.push({ docId: entry.docId, document: entry.document });
2071
+ else inserts.push(entry.document);
2072
+ }
2073
+ for (const outcome of [
2074
+ inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
2075
+ updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
2076
+ ]) {
2077
+ if (outcome === null) continue;
2078
+ state.result.imported += outcome.succeeded.length;
2079
+ for (const failure of outcome.failed) {
2080
+ const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
2081
+ refuse3(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
2082
+ }
1720
2083
  }
1721
- if (config.required !== void 0) {
1722
- metadata.required = config.required;
2084
+ }
2085
+ async function takeLine(state, entry) {
2086
+ if (!("text" in entry)) {
2087
+ const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
2088
+ refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
2089
+ return;
1723
2090
  }
1724
- if (config.vectorPromotion !== void 0) {
1725
- metadata.vectorPromotion = config.vectorPromotion;
2091
+ if (entry.text.length === 0) return;
2092
+ let document;
2093
+ try {
2094
+ document = JSON.parse(entry.text);
2095
+ } catch {
2096
+ refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
2097
+ return;
1726
2098
  }
1727
- if (config.patternValueLimit !== void 0) {
1728
- metadata.patternValueLimit = config.patternValueLimit;
2099
+ if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
2100
+ refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
2101
+ return;
1729
2102
  }
1730
- if (config.indexUuid !== void 0) {
1731
- metadata.indexUuid = config.indexUuid;
2103
+ try {
2104
+ untagDocumentLine(entry.text, document);
2105
+ } catch (error) {
2106
+ if (!(error instanceof NarsilError)) throw error;
2107
+ refuse3(state, entry.line, error.code, error.message);
2108
+ return;
1732
2109
  }
1733
- if (config.synonyms !== void 0) {
1734
- metadata.synonyms = config.synonyms;
2110
+ if (state.batchIds.has(document.id)) await flush(state);
2111
+ state.batch.push({ line: entry.line, docId: document.id, document });
2112
+ state.batchIds.add(document.id);
2113
+ if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
2114
+ }
2115
+ async function readDocumentExport(engine, indexName, source) {
2116
+ const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
2117
+ const header = await lines.next();
2118
+ await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
2119
+ const state = {
2120
+ engine,
2121
+ indexName,
2122
+ batch: [],
2123
+ batchIds: /* @__PURE__ */ new Set(),
2124
+ result: { imported: 0, refusedCount: 0, refused: [] },
2125
+ highestReported: 0
2126
+ };
2127
+ for await (const entry of lines) {
2128
+ await takeLine(state, entry);
2129
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
1735
2130
  }
1736
- if (config.heldPartitions !== void 0) {
1737
- metadata.heldPartitions = config.heldPartitions;
2131
+ await flush(state);
2132
+ state.result.refused.sort((a, b) => a.line - b.line);
2133
+ return state.result;
2134
+ }
2135
+
2136
+ // src/engine/export/json-line.ts
2137
+ function refuse4(path, kind, docId) {
2138
+ throw new NarsilError(
2139
+ ErrorCodes.DOC_VALIDATION_FAILED,
2140
+ `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
2141
+ { docId, field: path }
2142
+ );
2143
+ }
2144
+ function encodeNumber(value, path, target) {
2145
+ if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
2146
+ if (!target.tagged) refuse4(path, String(value), target.docId);
2147
+ return `{"${NUMBER_TAG}":"${String(value)}"}`;
2148
+ }
2149
+ function encodeDate(value, path, target) {
2150
+ const milliseconds = value.getTime();
2151
+ if (!target.tagged || Number.isNaN(milliseconds)) refuse4(path, "a date", target.docId);
2152
+ const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
2153
+ const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
2154
+ return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
2155
+ }
2156
+ function encodeBytes(value, path, target) {
2157
+ if (!target.tagged) refuse4(path, "bytes", target.docId);
2158
+ return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
2159
+ }
2160
+ function encodeList(value, path, target) {
2161
+ target.parts.push("[");
2162
+ for (let at = 0; at < value.length; at++) {
2163
+ if (at > 0) target.parts.push(",");
2164
+ encodeValue(value[at], `${path}[${at}]`, target);
2165
+ }
2166
+ target.parts.push("]");
2167
+ }
2168
+ function encodeMap(value, path, target) {
2169
+ const proto = Object.getPrototypeOf(value);
2170
+ if (proto !== Object.prototype && proto !== null) refuse4(path, "a value other than a plain object", target.docId);
2171
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
2172
+ const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
2173
+ target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
2174
+ for (let at = 0; at < entries.length; at++) {
2175
+ const [key, entry] = entries[at];
2176
+ if (at > 0) target.parts.push(",");
2177
+ target.parts.push(JSON.stringify(key), ":");
2178
+ encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
2179
+ }
2180
+ target.parts.push(wrapped ? "}}" : "}");
2181
+ }
2182
+ function encodeValue(value, path, target) {
2183
+ if (value === null) {
2184
+ target.parts.push("null");
2185
+ return;
1738
2186
  }
1739
- if (config.sync !== void 0) metadata.sync = config.sync;
1740
- if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
1741
- if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
1742
- if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
1743
- return metadata;
2187
+ switch (typeof value) {
2188
+ case "string":
2189
+ target.parts.push(JSON.stringify(value));
2190
+ return;
2191
+ case "boolean":
2192
+ target.parts.push(value ? "true" : "false");
2193
+ return;
2194
+ case "number":
2195
+ target.parts.push(encodeNumber(value, path, target));
2196
+ return;
2197
+ case "object":
2198
+ break;
2199
+ default:
2200
+ refuse4(path, `a ${typeof value}`, target.docId);
2201
+ }
2202
+ if (Array.isArray(value)) encodeList(value, path, target);
2203
+ else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
2204
+ else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
2205
+ else encodeMap(value, path, target);
2206
+ }
2207
+ function encodeExactJson(value, docId) {
2208
+ const target = { docId, tagged: false, parts: [] };
2209
+ encodeValue(value, "", target);
2210
+ return target.parts.join("");
2211
+ }
2212
+ function encodeDocumentJson(document, docId) {
2213
+ const target = { docId, tagged: true, parts: [] };
2214
+ encodeValue(document, "", target);
2215
+ return target.parts.join("");
1744
2216
  }
1745
- function metadataConfigOf(entry) {
1746
- const embedding = entry.config.embedding ? {
1747
- fields: entry.config.embedding.fields,
1748
- ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
1749
- } : void 0;
1750
- return {
1751
- ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
1752
- ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
1753
- documentCount: entry.documentCount,
1754
- savedBytes: entry.savedBytes,
1755
- partitionCount: entry.partitionCount,
1756
- schema: flattenSchema(entry.config.schema),
1757
- language: entry.language.name,
1758
- k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
1759
- b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
1760
- ...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
1761
- ...embedding !== void 0 ? { embedding } : {},
1762
- surfaceForms: entry.config.surfaceForms !== false,
1763
- ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
1764
- ...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
1765
- ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
1766
- ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
1767
- ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
1768
- ...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
1769
- ...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
1770
- ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
1771
- ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
1772
- ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
1773
- ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
1774
- ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
1775
- ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
1776
- ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
1777
- ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
1778
- ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
1779
- ...syncMetadataOf(entry),
1780
- ...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
1781
- };
2217
+
2218
+ // src/engine/export/paged-lines.ts
2219
+ async function coldFieldOf(checkpoint, field) {
2220
+ const { directory } = checkpoint;
2221
+ if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
2222
+ if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
2223
+ const fieldType = checkpoint.manifest.schema[field.fieldPath];
2224
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
2225
+ if (dimension === null) return null;
2226
+ const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
2227
+ return openColdVectorField({
2228
+ fieldPath: field.fieldPath,
2229
+ dimension,
2230
+ metric: "cosine",
2231
+ codeBits: null,
2232
+ files: await Promise.all(field.files.map(withPath)),
2233
+ graphParts: await Promise.all(field.graphParts.map(withPath)),
2234
+ resolvePartition: () => void 0
2235
+ });
1782
2236
  }
1783
- function syncMetadataOf(entry) {
1784
- const sync = entry.sync;
1785
- if (sync === null) return {};
2237
+ async function openFileVectorReads(checkpoint, wholeFileReads) {
2238
+ const fields = /* @__PURE__ */ new Map();
2239
+ for (const field of checkpoint.manifest.vectors) {
2240
+ const opened = await coldFieldOf(checkpoint, field);
2241
+ if (opened !== null) fields.set(field.fieldPath, opened);
2242
+ }
2243
+ const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
1786
2244
  return {
1787
- sync: {
1788
- enabled: true,
1789
- ...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
1790
- ...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
2245
+ async vectorOf(fieldPath, docId) {
2246
+ const field = fields.get(fieldPath);
2247
+ if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
2248
+ const number = field.numberOf(docId);
2249
+ return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
1791
2250
  },
1792
- syncActive: sync.active,
1793
- syncEpoch: sync.epoch
1794
- };
1795
- }
1796
- function durableMetadataConfigOf(entry) {
1797
- const terms = metadataConfigOf(entry);
1798
- const durable = entry.durableAnalysis;
1799
- if (durable === null) return terms;
1800
- const {
1801
- language,
1802
- tokenizer,
1803
- stopWords,
1804
- stopWordList,
1805
- analysisRevision: _analysisRevision,
1806
- arrayPositionGap: _arrayPositionGap,
1807
- analysisGeneration: _analysisGeneration,
1808
- ...rest
1809
- } = terms;
1810
- return {
1811
- ...rest,
1812
- language: durable.language,
1813
- ...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
1814
- ...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
1815
- ...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
1816
- ...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
1817
- ...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
1818
- ...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
1819
- pendingAnalysis: {
1820
- generation: entry.analysisGeneration,
1821
- language,
1822
- ...tokenizer !== void 0 ? { tokenizer } : {},
1823
- ...stopWords !== void 0 ? { stopWords } : {},
1824
- ...stopWordList !== void 0 ? { stopWordList } : {}
2251
+ close() {
2252
+ for (const field of fields.values()) field.close();
2253
+ fallback?.close();
1825
2254
  }
1826
2255
  };
1827
2256
  }
1828
- function indexMetadataOf(indexName, entry, partitionCount) {
1829
- return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
1830
- }
1831
-
1832
- // src/engine/snapshot/stored-schema.ts
1833
- function isRecord2(value) {
1834
- return typeof value === "object" && value !== null && !Array.isArray(value);
1835
- }
1836
- function conflictingStoredPath(path) {
1837
- return new NarsilError(
1838
- ErrorCodes.SCHEMA_INVALID_TYPE,
1839
- `Field path "${path}" is declared both as a field and as an object of fields`,
1840
- { path }
1841
- );
1842
- }
1843
- function defineOwn(level, key, value) {
1844
- Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
1845
- }
1846
- function placeStoredField(nested, path, type) {
1847
- const segments = path.split(".");
1848
- let level = nested;
1849
- for (let at = 0; at < segments.length; at++) {
1850
- const segment = segments[at];
1851
- const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
1852
- if (at === segments.length - 1) {
1853
- if (held !== void 0) throw conflictingStoredPath(path);
1854
- defineOwn(level, segment, type);
1855
- return;
2257
+ async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
2258
+ const { directory } = checkpoint;
2259
+ if (directory === void 0) return;
2260
+ const scans = [];
2261
+ try {
2262
+ for (const ref of partition.segments) {
2263
+ const paged = await isPagedSegmentFile(directory, ref.key);
2264
+ scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
1856
2265
  }
1857
- if (held === void 0) {
1858
- const child = {};
1859
- defineOwn(level, segment, child);
1860
- level = child;
1861
- } else if (isRecord2(held)) {
1862
- level = held;
1863
- } else {
1864
- throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
2266
+ for (let at = 0; at < scans.length; at++) {
2267
+ const scan = scans[at];
2268
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) {
2269
+ const docId = scan.idAt(ordinal);
2270
+ if (!countsAsLatestHolder(scans, at, docId)) continue;
2271
+ const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
2272
+ document.id = docId;
2273
+ for (const field of checkpoint.manifest.vectors) {
2274
+ const vector = await vectors.vectorOf(field.fieldPath, docId);
2275
+ if (vector !== null) setNestedValue(document, field.fieldPath, vector);
2276
+ }
2277
+ yield `${encodeDocumentJson(document, docId)}
2278
+ `;
2279
+ }
1865
2280
  }
2281
+ } finally {
2282
+ for (const scan of scans) scan.close();
1866
2283
  }
1867
2284
  }
1868
- function placeStoredFields(nested, stored, prefix) {
1869
- for (const [key, type] of Object.entries(stored)) {
1870
- const path = prefix ? `${prefix}.${key}` : key;
1871
- if (isRecord2(type)) {
1872
- placeStoredFields(nested, type, path);
1873
- } else {
1874
- placeStoredField(nested, path, type);
2285
+
2286
+ // src/engine/export/write.ts
2287
+ var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
2288
+ var FLOAT32_BYTES = 4;
2289
+ var ENVELOPE_VERSION_OFFSET = 4;
2290
+ async function locateVectors(checkpoint) {
2291
+ const located = /* @__PURE__ */ new Map();
2292
+ for (const field of checkpoint.manifest.vectors) {
2293
+ const byDocId = /* @__PURE__ */ new Map();
2294
+ for (const ref of field.files) {
2295
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
2296
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
2297
+ what: `vector file "${ref.key}"`,
2298
+ allowsCompression: false
2299
+ });
2300
+ const decoded = decodeVectorFilePayload(decode(payload));
2301
+ for (let position = 0; position < decoded.docIds.length; position++) {
2302
+ if (isDead(ref.dead, position)) continue;
2303
+ byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
2304
+ }
1875
2305
  }
2306
+ located.set(field.fieldPath, byDocId);
1876
2307
  }
2308
+ return located;
1877
2309
  }
1878
- function nestedSchemaOf(stored) {
1879
- if (!isRecord2(stored)) return stored;
1880
- const nested = {};
1881
- placeStoredFields(nested, stored, "");
1882
- return nested;
2310
+ async function readVector(checkpoint, location) {
2311
+ const width = location.dimension * FLOAT32_BYTES;
2312
+ const offset = location.ref.bytes - (location.ref.count - location.position) * width;
2313
+ return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
1883
2314
  }
1884
-
1885
- // src/persistence/durability/complete-graphs.ts
1886
- async function whileGraphsStayComplete(vectorIndexes, capture2) {
1887
- const letBuildsIn = [];
2315
+ async function readSegment(checkpoint, ref) {
2316
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
2317
+ envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
2318
+ what: `segment "${ref.key}"`,
2319
+ allowsCompression: true
2320
+ });
2321
+ return decodeKeywordSegment(payload, false);
2322
+ }
2323
+ var decoder = new TextDecoder();
2324
+ function idAt(contents, ordinal) {
2325
+ const ids = contents.arrays.idTable;
2326
+ return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
2327
+ }
2328
+ async function winnersOf(checkpoint, partition) {
2329
+ const winners = /* @__PURE__ */ new Map();
2330
+ for (let at = 0; at < partition.segments.length; at++) {
2331
+ const contents = await readSegment(checkpoint, partition.segments[at]);
2332
+ for (const docId of contents.tombstones) winners.delete(docId);
2333
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
2334
+ }
2335
+ return winners;
2336
+ }
2337
+ async function wholeSegmentScan(checkpoint, ref) {
2338
+ const contents = await readSegment(checkpoint, ref);
2339
+ const table = contents.arrays.documentTable;
2340
+ const ids = /* @__PURE__ */ new Set();
2341
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
2342
+ return {
2343
+ count: contents.arrays.documentCount,
2344
+ schema: contents.schema,
2345
+ tombstones: new Set(contents.tombstones),
2346
+ idAt: (ordinal) => idAt(contents, ordinal),
2347
+ holds: (docId) => ids.has(docId),
2348
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
2349
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
2350
+ table.discardedFields
2351
+ ),
2352
+ close: () => void 0
2353
+ };
2354
+ }
2355
+ async function wholeFileVectorReads(checkpoint) {
2356
+ const located = await locateVectors(checkpoint);
2357
+ return {
2358
+ async vectorOf(fieldPath, docId) {
2359
+ const location = located.get(fieldPath)?.get(docId);
2360
+ return location === void 0 ? null : readVector(checkpoint, location);
2361
+ },
2362
+ close: () => located.clear()
2363
+ };
2364
+ }
2365
+ async function* pagedDocumentLines(checkpoint) {
2366
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
1888
2367
  try {
1889
- for (const fieldPath of [...vectorIndexes.keys()].sort()) {
1890
- const vectorIndex = vectorIndexes.get(fieldPath);
1891
- if (vectorIndex !== void 0) letBuildsIn.push(await vectorIndex.holdGraphComplete());
2368
+ for (const partition of checkpoint.manifest.partitions) {
2369
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
1892
2370
  }
1893
- return await capture2();
1894
2371
  } finally {
1895
- for (const release of letBuildsIn) release();
2372
+ vectors.close();
1896
2373
  }
1897
2374
  }
1898
-
1899
- // src/serialization/crc32-combine.ts
1900
- var IEEE_POLYNOMIAL = 3988292384;
1901
- var CRC_BITS = 32;
1902
- var zeroByteOperators = [];
1903
- function timesVector(operator, vector) {
1904
- let sum = 0;
1905
- let rest = vector >>> 0;
1906
- for (let row = 0; rest !== 0; row++, rest >>>= 1) {
1907
- if (rest & 1) sum ^= operator[row];
2375
+ async function anyPartitionIsPaged(checkpoint) {
2376
+ for (const partition of checkpoint.manifest.partitions) {
2377
+ for (const ref of partition.segments) {
2378
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
2379
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
2380
+ }
1908
2381
  }
1909
- return sum >>> 0;
1910
- }
1911
- function squared(operator) {
1912
- const result = new Uint32Array(CRC_BITS);
1913
- for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
1914
- return result;
1915
- }
1916
- function firstOperator() {
1917
- let oneZeroBit = new Uint32Array(CRC_BITS);
1918
- oneZeroBit[0] = IEEE_POLYNOMIAL;
1919
- for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
1920
- for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
1921
- return oneZeroBit;
2382
+ return false;
1922
2383
  }
1923
- function operatorForZeroBytesPower(power) {
1924
- if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
1925
- while (zeroByteOperators.length <= power) {
1926
- zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
2384
+ async function* documentLines(checkpoint) {
2385
+ if (await anyPartitionIsPaged(checkpoint)) {
2386
+ yield* pagedDocumentLines(checkpoint);
2387
+ return;
1927
2388
  }
1928
- return zeroByteOperators[power];
1929
- }
1930
- function crc32Combine(first, second, secondByteLength) {
1931
- let crc = first >>> 0;
1932
- let rest = secondByteLength;
1933
- for (let power = 0; rest > 0; power++) {
1934
- if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
1935
- rest = Math.floor(rest / 2);
2389
+ const vectors = await locateVectors(checkpoint);
2390
+ for (const partition of checkpoint.manifest.partitions) {
2391
+ const winners = await winnersOf(checkpoint, partition);
2392
+ for (let at = 0; at < partition.segments.length; at++) {
2393
+ const contents = await readSegment(checkpoint, partition.segments[at]);
2394
+ const table = contents.arrays.documentTable;
2395
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
2396
+ const docId = idAt(contents, ordinal);
2397
+ if (winners.get(docId) !== at) continue;
2398
+ const stored = decodeStoredDocument(
2399
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
2400
+ table.discardedFields
2401
+ );
2402
+ const document = { id: docId, ...stored };
2403
+ document.id = docId;
2404
+ for (const [fieldPath, located] of vectors) {
2405
+ const location = located.get(docId);
2406
+ if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
2407
+ }
2408
+ yield `${encodeDocumentJson(document, docId)}
2409
+ `;
2410
+ }
2411
+ }
1936
2412
  }
1937
- return (crc ^ second) >>> 0;
1938
2413
  }
1939
-
1940
- // src/persistence/paged/vector-bytes.ts
1941
- var BIN8 = 196;
1942
- var BIN16 = 197;
1943
- var BIN32 = 198;
1944
- var ARRAY16 = 220;
1945
- var ARRAY32 = 221;
1946
- var FIXMAP = 128;
1947
- var FIXARRAY = 144;
1948
- var FIXED_LIST_MAX = 15;
1949
- var FIXSTR = 160;
1950
- var FIXSTR_MAX_LENGTH = 31;
1951
- var ASCII_MAX = 127;
1952
- var UTF8_MAX_BYTES_PER_UNIT = 3;
1953
- var STRING_HEADER_MAX_BYTES = 5;
1954
- var BIN8_MAX_BYTES = 255;
1955
- var UINT16_MAX = 65535;
1956
- function binHeader(byteLength) {
1957
- if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
1958
- if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1959
- const header = new Uint8Array(5);
1960
- header[0] = BIN32;
1961
- new DataView(header.buffer).setUint32(1, byteLength, false);
1962
- return header;
2414
+ async function* exportLines2(header, checkpoint, positions) {
2415
+ const exportHeader = { format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header };
2416
+ yield `${encodeExactJson(positions === null ? exportHeader : { ...exportHeader, ...positions }, "")}
2417
+ `;
2418
+ yield* documentLines(checkpoint);
2419
+ }
2420
+ function documentExportStream(header, checkpoint, positions = null) {
2421
+ const lines = exportLines2(header, checkpoint, positions);
2422
+ const encoder = new TextEncoder();
2423
+ let released = false;
2424
+ const release = () => {
2425
+ if (released) return;
2426
+ released = true;
2427
+ checkpoint.release();
2428
+ };
2429
+ return new ReadableStream({
2430
+ async pull(controller) {
2431
+ try {
2432
+ let text = "";
2433
+ while (text.length < EXPORT_CHUNK_BYTES) {
2434
+ const step = await lines.next();
2435
+ if (step.done) {
2436
+ if (text.length > 0) controller.enqueue(encoder.encode(text));
2437
+ release();
2438
+ controller.close();
2439
+ return;
2440
+ }
2441
+ text += step.value;
2442
+ }
2443
+ controller.enqueue(encoder.encode(text));
2444
+ } catch (error) {
2445
+ release();
2446
+ controller.error(error);
2447
+ }
2448
+ },
2449
+ async cancel() {
2450
+ release();
2451
+ await lines.return(void 0);
2452
+ }
2453
+ });
2454
+ }
2455
+
2456
+ // src/partitioning/saved-partitions.ts
2457
+ function requireSavedPartitionCount(partitionCount, source, indexName) {
2458
+ if (partitionCount <= MAX_PARTITIONS) return;
2459
+ throw new NarsilError(
2460
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2461
+ `The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
2462
+ { indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
2463
+ );
2464
+ }
2465
+ function highestPartitionIdOf(partitions) {
2466
+ let highest = partitions.length - 1;
2467
+ for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
2468
+ return highest;
2469
+ }
2470
+
2471
+ // src/persistence/paged/read-ahead.ts
2472
+ var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
2473
+ function rangeOf(path, section) {
2474
+ return { path, offset: HEADER_SIZE + section.offset, length: section.length };
2475
+ }
2476
+ async function readRangesAhead(ranges) {
2477
+ let rangeAt = 0;
2478
+ let doneInRange = 0;
2479
+ function nextChunk() {
2480
+ while (rangeAt < ranges.length && doneInRange >= ranges[rangeAt].length) {
2481
+ rangeAt += 1;
2482
+ doneInRange = 0;
2483
+ }
2484
+ if (rangeAt >= ranges.length) return null;
2485
+ const { path, offset, length } = ranges[rangeAt];
2486
+ const chunk = { path, offset: offset + doneInRange, length: Math.min(READ_AHEAD_CHUNK_BYTES, length - doneInRange) };
2487
+ doneInRange += chunk.length;
2488
+ return chunk;
2489
+ }
2490
+ const scratch = new Uint8Array(READ_AHEAD_CHUNK_BYTES);
2491
+ async function readChunks() {
2492
+ for (let chunk = nextChunk(); chunk !== null; chunk = nextChunk()) {
2493
+ const target = scratch.subarray(0, chunk.length);
2494
+ if (!await readFileRangeLater(chunk.path, chunk.offset, target)) return;
2495
+ }
2496
+ }
2497
+ await Promise.all(Array.from({ length: READS_AHEAD_AT_ONCE }, readChunks));
2498
+ }
2499
+ async function headOf(path, ref) {
2500
+ const { headOffset, headLength } = ref;
2501
+ if (headOffset === void 0 || headLength === void 0) return null;
2502
+ const head = new Uint8Array(headLength);
2503
+ if (!await readFileRangeLater(path, HEADER_SIZE + headOffset, head)) return null;
2504
+ return decode(head);
2505
+ }
2506
+ async function levelsOfPart(directory, ref) {
2507
+ const path = await directory.pathOf(ref.key);
2508
+ const head = await headOf(path, ref);
2509
+ const graphs = head?.graphs;
2510
+ if (!Array.isArray(graphs) || graphs.length !== 1) return [];
2511
+ const { levels } = graphs[0];
2512
+ return decodeSections([levels], ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES).map(
2513
+ (section) => rangeOf(path, section)
2514
+ );
2515
+ }
2516
+ async function residentSectionsOfSegment(directory, ref) {
2517
+ if (!await isPagedSegmentFile(directory, ref.key)) return [];
2518
+ const listing = await pagedListingOf(directory, ref);
2519
+ return residentSectionsOf(listing).map((section) => rangeOf(listing.path, section));
2520
+ }
2521
+ async function startReadingOpeningSectionsAhead(directory, manifest) {
2522
+ try {
2523
+ const groups = await Promise.all([
2524
+ ...manifest.partitions.flatMap(
2525
+ (partition) => partition.segments.map((ref) => residentSectionsOfSegment(directory, ref))
2526
+ ),
2527
+ ...manifest.vectors.flatMap((field) => field.graphParts.map((ref) => levelsOfPart(directory, ref)))
2528
+ ]);
2529
+ void readRangesAhead(groups.flat());
2530
+ } catch {
2531
+ return;
2532
+ }
2533
+ }
2534
+
2535
+ // src/engine/changes/operations.ts
2536
+ function invalidLimit(name, value) {
2537
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The changes setting ${name} must be a positive whole number`, {
2538
+ [name]: String(value)
2539
+ });
2540
+ }
2541
+ function requireChangesSetting(setting) {
2542
+ if (setting === null || typeof setting !== "object" || Array.isArray(setting)) {
2543
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting must be an object");
2544
+ }
2545
+ const { enabled, previousDocument, historyMaxAgeMs, historyMaxBytes } = setting;
2546
+ if (typeof enabled !== "boolean") {
2547
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting needs enabled set to true or false");
2548
+ }
2549
+ if (previousDocument !== void 0 && typeof previousDocument !== "boolean") {
2550
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting previousDocument must be true or false");
2551
+ }
2552
+ const isLimit = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
2553
+ if (historyMaxAgeMs !== void 0 && !isLimit(historyMaxAgeMs)) invalidLimit("historyMaxAgeMs", historyMaxAgeMs);
2554
+ if (historyMaxBytes !== void 0 && !isLimit(historyMaxBytes)) invalidLimit("historyMaxBytes", historyMaxBytes);
2555
+ return {
2556
+ enabled,
2557
+ ...previousDocument !== void 0 ? { previousDocument } : {},
2558
+ ...historyMaxAgeMs !== void 0 ? { historyMaxAgeMs } : {},
2559
+ ...historyMaxBytes !== void 0 ? { historyMaxBytes } : {}
2560
+ };
2561
+ }
2562
+ function changesStorageOf(core, indexName) {
2563
+ const storage = core.durability?.manager.syncStorage?.();
2564
+ if (storage === void 0) {
2565
+ throw new NarsilError(
2566
+ ErrorCodes.CHANGES_DURABILITY_REQUIRED,
2567
+ `Index "${indexName}" needs write-ahead log durability for change capture, because the feed reads the log`,
2568
+ { indexName }
2569
+ );
2570
+ }
2571
+ return storage;
2572
+ }
2573
+ function changesStatusOf(changes) {
2574
+ if (changes === null) return { enabled: false, previousDocument: false, epoch: null };
2575
+ return {
2576
+ enabled: true,
2577
+ previousDocument: changes.previousDocument,
2578
+ ...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
2579
+ ...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {},
2580
+ epoch: changes.epoch
2581
+ };
2582
+ }
2583
+ function nextState(setting, epoch) {
2584
+ return {
2585
+ previousDocument: setting.previousDocument === true,
2586
+ ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
2587
+ ...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
2588
+ epoch
2589
+ };
2590
+ }
2591
+ async function turnOff(core, indexName, storage) {
2592
+ const { sync, changes } = core.requireIndex(indexName);
2593
+ if (changes === null) return;
2594
+ await persistEpochStates(core, indexName, { sync, changes: null });
2595
+ if (sync?.active !== true) await storage.release(indexName);
2596
+ emitSyncChange(core.indexRegistry, { kind: "changes-off", indexName });
2597
+ }
2598
+ async function turnOn(core, indexName, storage, setting) {
2599
+ const { sync, changes } = core.requireIndex(indexName);
2600
+ if (changes !== null) {
2601
+ await persistEpochStates(core, indexName, { sync, changes: nextState(setting, changes.epoch) });
2602
+ if (changes.previousDocument && setting.previousDocument !== true) {
2603
+ emitSyncChange(core.indexRegistry, { kind: "previous-document-off", indexName });
2604
+ }
2605
+ return;
2606
+ }
2607
+ if (sync !== null) {
2608
+ await persistEpochStates(core, indexName, { sync, changes: nextState(setting, sync.epoch) });
2609
+ return;
2610
+ }
2611
+ await storage.release(indexName);
2612
+ await persistEpochStates(core, indexName, { sync: null, changes: nextState(setting, newSyncEpoch()) });
2613
+ }
2614
+ function updateIndexChanges(core, indexName, value) {
2615
+ const setting = requireChangesSetting(value);
2616
+ core.requireIndex(indexName);
2617
+ const storage = changesStorageOf(core, indexName);
2618
+ return withOpenIndex(
2619
+ core,
2620
+ indexName,
2621
+ () => oneSyncChangeAtATime(
2622
+ core.indexRegistry,
2623
+ indexName,
2624
+ () => setting.enabled ? turnOn(core, indexName, storage, setting) : turnOff(core, indexName, storage)
2625
+ )
2626
+ );
2627
+ }
2628
+ async function updateChangesOf(core, indexName, setting) {
2629
+ core.guardShutdown();
2630
+ core.requireIndex(indexName);
2631
+ await updateIndexChanges(core, indexName, setting);
2632
+ }
2633
+ function changesOf(core, indexName) {
2634
+ core.guardShutdown();
2635
+ return changesStatusOf(core.requireIndex(indexName).changes);
2636
+ }
2637
+
2638
+ // src/persistence/durability/complete-graphs.ts
2639
+ async function whileGraphsStayComplete(vectorIndexes, capture2) {
2640
+ const letBuildsIn = [];
2641
+ try {
2642
+ for (const fieldPath of [...vectorIndexes.keys()].sort()) {
2643
+ const vectorIndex = vectorIndexes.get(fieldPath);
2644
+ if (vectorIndex !== void 0) letBuildsIn.push(await vectorIndex.holdGraphComplete());
2645
+ }
2646
+ return await capture2();
2647
+ } finally {
2648
+ for (const release of letBuildsIn) release();
2649
+ }
2650
+ }
2651
+
2652
+ // src/serialization/crc32-combine.ts
2653
+ var IEEE_POLYNOMIAL = 3988292384;
2654
+ var CRC_BITS = 32;
2655
+ var zeroByteOperators = [];
2656
+ function timesVector(operator, vector) {
2657
+ let sum = 0;
2658
+ let rest = vector >>> 0;
2659
+ for (let row = 0; rest !== 0; row++, rest >>>= 1) {
2660
+ if (rest & 1) sum ^= operator[row];
2661
+ }
2662
+ return sum >>> 0;
2663
+ }
2664
+ function squared(operator) {
2665
+ const result = new Uint32Array(CRC_BITS);
2666
+ for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
2667
+ return result;
2668
+ }
2669
+ function firstOperator() {
2670
+ let oneZeroBit = new Uint32Array(CRC_BITS);
2671
+ oneZeroBit[0] = IEEE_POLYNOMIAL;
2672
+ for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
2673
+ for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
2674
+ return oneZeroBit;
2675
+ }
2676
+ function operatorForZeroBytesPower(power) {
2677
+ if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
2678
+ while (zeroByteOperators.length <= power) {
2679
+ zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
2680
+ }
2681
+ return zeroByteOperators[power];
2682
+ }
2683
+ function crc32Combine(first, second, secondByteLength) {
2684
+ let crc = first >>> 0;
2685
+ let rest = secondByteLength;
2686
+ for (let power = 0; rest > 0; power++) {
2687
+ if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
2688
+ rest = Math.floor(rest / 2);
2689
+ }
2690
+ return (crc ^ second) >>> 0;
2691
+ }
2692
+
2693
+ // src/persistence/paged/vector-bytes.ts
2694
+ var BIN8 = 196;
2695
+ var BIN16 = 197;
2696
+ var BIN32 = 198;
2697
+ var ARRAY16 = 220;
2698
+ var ARRAY32 = 221;
2699
+ var FIXMAP = 128;
2700
+ var FIXARRAY = 144;
2701
+ var FIXED_LIST_MAX = 15;
2702
+ var FIXSTR = 160;
2703
+ var FIXSTR_MAX_LENGTH = 31;
2704
+ var ASCII_MAX = 127;
2705
+ var UTF8_MAX_BYTES_PER_UNIT = 3;
2706
+ var STRING_HEADER_MAX_BYTES = 5;
2707
+ var BIN8_MAX_BYTES = 255;
2708
+ var UINT16_MAX = 65535;
2709
+ function binHeader(byteLength) {
2710
+ if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
2711
+ if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
2712
+ const header = new Uint8Array(5);
2713
+ header[0] = BIN32;
2714
+ new DataView(header.buffer).setUint32(1, byteLength, false);
2715
+ return header;
1963
2716
  }
1964
2717
  function arrayHeader(count) {
1965
2718
  if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
@@ -2300,7 +3053,7 @@ async function writeLiveVectors(input) {
2300
3053
  }
2301
3054
 
2302
3055
  // src/persistence/archive/memory-store.ts
2303
- function joined(chunks) {
3056
+ function joined2(chunks) {
2304
3057
  let total = 0;
2305
3058
  for (const chunk of chunks) total += chunk.byteLength;
2306
3059
  const bytes = new Uint8Array(total);
@@ -2317,7 +3070,7 @@ function memoryCheckpointStore() {
2317
3070
  files,
2318
3071
  read: async (key) => files.get(key) ?? null,
2319
3072
  write: async (key, chunks) => {
2320
- files.set(key, joined(chunks));
3073
+ files.set(key, joined2(chunks));
2321
3074
  },
2322
3075
  remove: async (key) => {
2323
3076
  files.delete(key);
@@ -2358,150 +3111,549 @@ function partitionVectorPlan(vectorIndex, docIds) {
2358
3111
  };
2359
3112
  files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
2360
3113
  }
2361
- return {
2362
- unchanged: false,
2363
- storesHeads: false,
2364
- kept: [],
2365
- keptFiles: [],
2366
- newFiles: files.length,
2367
- graphParts: null,
2368
- signature: { revision: 0, graphSize: -1, calibration: null },
2369
- readNewFile: (index) => files[index],
2370
- release: () => void 0
2371
- };
3114
+ return {
3115
+ unchanged: false,
3116
+ storesHeads: false,
3117
+ kept: [],
3118
+ keptFiles: [],
3119
+ newFiles: files.length,
3120
+ graphParts: null,
3121
+ signature: { revision: 0, graphSize: -1, calibration: null },
3122
+ readNewFile: (index) => files[index],
3123
+ release: () => void 0
3124
+ };
3125
+ }
3126
+ function capture(input) {
3127
+ const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
3128
+ const { indexName } = input;
3129
+ const writeRecords = input.writeRecords ?? null;
3130
+ const targets = partitionIds.map((partitionId) => {
3131
+ const target = {
3132
+ partitionId,
3133
+ lastSeqNo: input.appliedSeqNo(partitionId),
3134
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
3135
+ };
3136
+ if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
3137
+ return target;
3138
+ });
3139
+ const contents = partitionIds.map(
3140
+ (partitionId) => partitionContentsOf(
3141
+ input.manager,
3142
+ partitionId,
3143
+ writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
3144
+ )
3145
+ );
3146
+ const plans = /* @__PURE__ */ new Map();
3147
+ const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
3148
+ for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
3149
+ plans.set(
3150
+ fieldPath,
3151
+ partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
3152
+ );
3153
+ }
3154
+ return { targets, contents, plans };
3155
+ }
3156
+ async function buildArchiveFromMemory(input) {
3157
+ const captured = input.partitionId === null ? await whileGraphsStayComplete(input.vectorIndexes, () => capture(input)) : capture(input);
3158
+ const store = memoryCheckpointStore();
3159
+ const written = await writeCheckpointSegments({
3160
+ store,
3161
+ metadata: input.metadata,
3162
+ targets: captured.targets,
3163
+ compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3164
+ prior: null,
3165
+ wholePartitions: captured.contents,
3166
+ keepsWriteRecords: input.writeRecords != null
3167
+ });
3168
+ const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
3169
+ const refs = vectors.refs;
3170
+ await commitCheckpointManifest(store, input.metadata, written, refs);
3171
+ const manifestBytes = store.files.get(manifestKey(input.indexName));
3172
+ const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName, input.partitionId);
3173
+ if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
3174
+ throw new NarsilError(
3175
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
3176
+ `The archive of index "${input.indexName}" holds no manifest`
3177
+ );
3178
+ }
3179
+ return {
3180
+ manifestBytes,
3181
+ manifest: decoded.manifest,
3182
+ readFile: async (ref) => {
3183
+ const bytes = store.files.get(ref.key);
3184
+ if (bytes === void 0) {
3185
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The archive file "${ref.key}" is missing`, {
3186
+ key: ref.key
3187
+ });
3188
+ }
3189
+ return bytes;
3190
+ },
3191
+ readRange: async (ref, offset, length) => {
3192
+ const bytes = store.files.get(ref.key);
3193
+ if (bytes === void 0 || offset + length > bytes.byteLength) {
3194
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive file "${ref.key}" is missing`, {
3195
+ key: ref.key
3196
+ });
3197
+ }
3198
+ return bytes.subarray(offset, offset + length);
3199
+ },
3200
+ release: () => store.files.clear()
3201
+ };
3202
+ }
3203
+
3204
+ // src/persistence/archive/write.ts
3205
+ function entryChunks(name, bytes) {
3206
+ return [...tarEntryHeaders(name, bytes.byteLength), bytes, tarPadding(bytes.byteLength)].filter(
3207
+ (chunk) => chunk.byteLength > 0
3208
+ );
3209
+ }
3210
+ async function* fileEntryChunks(source, name, ref) {
3211
+ yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
3212
+ for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
3213
+ yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
3214
+ }
3215
+ const padding = tarPadding(ref.bytes);
3216
+ if (padding.byteLength > 0) yield padding;
3217
+ }
3218
+ async function* archiveChunks(source) {
3219
+ yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
3220
+ yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
3221
+ const prefix = `${source.sourceIndexName}/`;
3222
+ for (const ref of manifestFileRefs(source.manifest)) {
3223
+ yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
3224
+ }
3225
+ if (source.trailingEntries !== void 0) {
3226
+ for await (const entry of source.trailingEntries()) yield* entryChunks(entry.name, entry.bytes);
3227
+ }
3228
+ yield tarEnd();
3229
+ }
3230
+ function archiveStream(source) {
3231
+ const chunks = archiveChunks(source);
3232
+ let released = false;
3233
+ const release = () => {
3234
+ if (released) return;
3235
+ released = true;
3236
+ source.release();
3237
+ };
3238
+ return new ReadableStream({
3239
+ async pull(controller) {
3240
+ try {
3241
+ const step = await chunks.next();
3242
+ if (step.done) {
3243
+ release();
3244
+ controller.close();
3245
+ return;
3246
+ }
3247
+ controller.enqueue(step.value);
3248
+ } catch (error) {
3249
+ release();
3250
+ controller.error(error);
3251
+ }
3252
+ },
3253
+ async cancel() {
3254
+ release();
3255
+ await chunks.return(void 0);
3256
+ }
3257
+ });
3258
+ }
3259
+
3260
+ // src/engine/durability-metadata.ts
3261
+ var VECTOR_TYPE = /^vector\[(\d+)\]$/;
3262
+ function vectorFieldsOf(schema, promotion) {
3263
+ const fields = {};
3264
+ for (const [fieldPath, type] of Object.entries(schema)) {
3265
+ const match = VECTOR_TYPE.exec(type);
3266
+ if (match === null) continue;
3267
+ const dimension = Number(match[1]);
3268
+ fields[fieldPath] = {
3269
+ dimension,
3270
+ metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
3271
+ quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
3272
+ };
3273
+ }
3274
+ return Object.keys(fields).length === 0 ? null : fields;
3275
+ }
3276
+ function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
3277
+ const metadata = {
3278
+ indexName,
3279
+ schema: config.schema,
3280
+ language: config.language,
3281
+ partitionCount: partitionCount ?? config.partitionCount,
3282
+ bm25Params: {
3283
+ k1: config.k1,
3284
+ b: config.b,
3285
+ ...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
3286
+ },
3287
+ createdAt: Date.now(),
3288
+ engineVersion: VERSION
3289
+ };
3290
+ metadata.documentCount = documentCount ?? config.documentCount;
3291
+ const recordedSavedBytes = savedBytes ?? config.savedBytes;
3292
+ if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
3293
+ const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
3294
+ if (vectorFields !== null) {
3295
+ metadata.vectorFields = vectorFields;
3296
+ }
3297
+ if (config.embedding) {
3298
+ metadata.embedding = config.embedding;
3299
+ }
3300
+ if (config.surfaceForms !== void 0) {
3301
+ metadata.surfaceForms = config.surfaceForms;
3302
+ }
3303
+ if (config.analysisRevision !== void 0) {
3304
+ metadata.analysisRevision = config.analysisRevision;
3305
+ }
3306
+ if (config.arrayPositionGap !== void 0) {
3307
+ metadata.arrayPositionGap = config.arrayPositionGap;
3308
+ }
3309
+ if (config.tokenizer !== void 0) {
3310
+ metadata.tokenizer = config.tokenizer;
3311
+ }
3312
+ if (config.stopWords !== void 0) {
3313
+ metadata.stopWords = config.stopWords;
3314
+ }
3315
+ if (config.stopWordList !== void 0) {
3316
+ metadata.stopWordList = config.stopWordList;
3317
+ }
3318
+ if (config.analysisGeneration !== void 0) {
3319
+ metadata.analysisGeneration = config.analysisGeneration;
3320
+ }
3321
+ if (config.pendingAnalysis !== void 0) {
3322
+ metadata.pendingAnalysis = config.pendingAnalysis;
3323
+ }
3324
+ if (config.partitionLimits !== void 0) {
3325
+ metadata.partitionLimits = config.partitionLimits;
3326
+ }
3327
+ if (config.defaultScoring !== void 0) {
3328
+ metadata.defaultScoring = config.defaultScoring;
3329
+ }
3330
+ if (config.trackPositions !== void 0) {
3331
+ metadata.trackPositions = config.trackPositions;
3332
+ }
3333
+ if (config.textStorage !== void 0) {
3334
+ metadata.textStorage = config.textStorage;
3335
+ }
3336
+ if (config.strict !== void 0) {
3337
+ metadata.strict = config.strict;
3338
+ }
3339
+ if (config.required !== void 0) {
3340
+ metadata.required = config.required;
3341
+ }
3342
+ if (config.vectorPromotion !== void 0) {
3343
+ metadata.vectorPromotion = config.vectorPromotion;
3344
+ }
3345
+ if (config.patternValueLimit !== void 0) {
3346
+ metadata.patternValueLimit = config.patternValueLimit;
3347
+ }
3348
+ if (config.indexUuid !== void 0) {
3349
+ metadata.indexUuid = config.indexUuid;
3350
+ }
3351
+ if (config.synonyms !== void 0) {
3352
+ metadata.synonyms = config.synonyms;
3353
+ }
3354
+ if (config.heldPartitions !== void 0) {
3355
+ metadata.heldPartitions = config.heldPartitions;
3356
+ }
3357
+ if (config.sync !== void 0) metadata.sync = config.sync;
3358
+ if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
3359
+ if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
3360
+ if (config.changes !== void 0) metadata.changes = config.changes;
3361
+ if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
3362
+ return metadata;
3363
+ }
3364
+ function metadataConfigOf(entry) {
3365
+ const embedding = entry.config.embedding ? {
3366
+ fields: entry.config.embedding.fields,
3367
+ ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
3368
+ } : void 0;
3369
+ return {
3370
+ ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
3371
+ ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
3372
+ documentCount: entry.documentCount,
3373
+ savedBytes: entry.savedBytes,
3374
+ partitionCount: entry.partitionCount,
3375
+ schema: flattenSchema(entry.config.schema),
3376
+ language: entry.language.name,
3377
+ k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
3378
+ b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
3379
+ ...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
3380
+ ...embedding !== void 0 ? { embedding } : {},
3381
+ surfaceForms: entry.config.surfaceForms !== false,
3382
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
3383
+ ...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
3384
+ ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
3385
+ ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
3386
+ ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
3387
+ ...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
3388
+ ...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
3389
+ ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
3390
+ ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
3391
+ ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
3392
+ ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
3393
+ ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
3394
+ ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
3395
+ ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
3396
+ ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
3397
+ ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
3398
+ ...syncMetadataOf(entry),
3399
+ ...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
3400
+ };
3401
+ }
3402
+ function syncMetadataOf(entry) {
3403
+ const { sync, changes } = entry;
3404
+ if (sync === null && changes === null) return {};
3405
+ return {
3406
+ ...sync === null ? {} : {
3407
+ sync: {
3408
+ enabled: true,
3409
+ ...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
3410
+ ...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
3411
+ },
3412
+ syncActive: sync.active
3413
+ },
3414
+ syncEpoch: sync?.epoch ?? changes?.epoch,
3415
+ ...changes === null ? {} : {
3416
+ changes: {
3417
+ enabled: true,
3418
+ ...changes.previousDocument ? { previousDocument: true } : {},
3419
+ ...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
3420
+ ...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {}
3421
+ }
3422
+ }
3423
+ };
3424
+ }
3425
+ function durableMetadataConfigOf(entry) {
3426
+ const terms = metadataConfigOf(entry);
3427
+ const durable = entry.durableAnalysis;
3428
+ if (durable === null) return terms;
3429
+ const {
3430
+ language,
3431
+ tokenizer,
3432
+ stopWords,
3433
+ stopWordList,
3434
+ analysisRevision: _analysisRevision,
3435
+ arrayPositionGap: _arrayPositionGap,
3436
+ analysisGeneration: _analysisGeneration,
3437
+ ...rest
3438
+ } = terms;
3439
+ return {
3440
+ ...rest,
3441
+ language: durable.language,
3442
+ ...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
3443
+ ...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
3444
+ ...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
3445
+ ...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
3446
+ ...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
3447
+ ...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
3448
+ pendingAnalysis: {
3449
+ generation: entry.analysisGeneration,
3450
+ language,
3451
+ ...tokenizer !== void 0 ? { tokenizer } : {},
3452
+ ...stopWords !== void 0 ? { stopWords } : {},
3453
+ ...stopWordList !== void 0 ? { stopWordList } : {}
3454
+ }
3455
+ };
3456
+ }
3457
+ function indexMetadataOf(indexName, entry, partitionCount) {
3458
+ return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
3459
+ }
3460
+
3461
+ // src/persistence/archive/file-entry.ts
3462
+ async function writeArchiveFile(store, key, chunks) {
3463
+ const { directory } = store;
3464
+ if (directory === void 0) {
3465
+ const held = [];
3466
+ for await (const chunk of chunks) held.push(chunk);
3467
+ await store.write(key, held);
3468
+ return;
3469
+ }
3470
+ const receiving = `${key}.receiving`;
3471
+ await directory.remove(receiving);
3472
+ const handle = await directory.appendHandle(receiving);
3473
+ try {
3474
+ for await (const chunk of chunks) await handle.append(chunk);
3475
+ await handle.sync();
3476
+ } finally {
3477
+ await handle.close();
3478
+ }
3479
+ await directory.move(receiving, key);
3480
+ }
3481
+
3482
+ // src/persistence/durability/segment/load-partition.ts
3483
+ function requireAgreedSchema(agreed, schema, key) {
3484
+ for (const [field, type] of Object.entries(schema)) {
3485
+ const existing = agreed[field];
3486
+ if (existing !== void 0 && existing !== type) {
3487
+ throw new NarsilError(
3488
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3489
+ `The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
3490
+ { key, field, existing, found: type }
3491
+ );
3492
+ }
3493
+ }
3494
+ }
3495
+ function requireListedCounts(ref, documentCount, tombstoneCount) {
3496
+ if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
3497
+ throw new NarsilError(
3498
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3499
+ `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
3500
+ {
3501
+ key: ref.key,
3502
+ documentCount,
3503
+ tombstoneCount,
3504
+ listedDocCount: ref.docCount,
3505
+ listedTombstoneCount: ref.tombstoneCount
3506
+ }
3507
+ );
3508
+ }
3509
+ function removeWhatTheSegmentReplaces(manager, segment, contents) {
3510
+ manager.beginBatchRemove();
3511
+ try {
3512
+ for (const docId of contents.tombstones) {
3513
+ if (manager.has(docId)) manager.remove(docId);
3514
+ }
3515
+ const ids = segment.arrays.idTable;
3516
+ for (let ordinal = 0; ordinal < ids.count; ordinal++) {
3517
+ const docId = ids.idAt(ordinal);
3518
+ if (manager.has(docId)) manager.remove(docId);
3519
+ }
3520
+ } finally {
3521
+ manager.endBatchRemove();
3522
+ }
3523
+ }
3524
+ function yieldToEventLoop2() {
3525
+ return new Promise((resolve) => setTimeout(resolve, 0));
3526
+ }
3527
+ function removeFromPartition(manager, partitionId, docIds) {
3528
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
3529
+ composite.beginBatch();
3530
+ try {
3531
+ for (const docId of docIds) {
3532
+ if (!composite.has(docId)) continue;
3533
+ composite.remove(docId, manager.config.schema, manager.language, insertOptions);
3534
+ docPartitionMap.delete(docId);
3535
+ }
3536
+ } finally {
3537
+ composite.endBatch();
3538
+ }
2372
3539
  }
2373
- function capture(input) {
2374
- const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
2375
- const { indexName } = input;
2376
- const writeRecords = input.writeRecords ?? null;
2377
- const targets = partitionIds.map((partitionId) => {
2378
- const target = {
2379
- partitionId,
2380
- lastSeqNo: input.appliedSeqNo(partitionId),
2381
- primaryTerm: SINGLE_NODE_PRIMARY_TERM
2382
- };
2383
- if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
2384
- return target;
2385
- });
2386
- const contents = partitionIds.map(
2387
- (partitionId) => partitionContentsOf(
2388
- input.manager,
2389
- partitionId,
2390
- writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
2391
- )
2392
- );
2393
- const plans = /* @__PURE__ */ new Map();
2394
- const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
2395
- for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
2396
- plans.set(
2397
- fieldPath,
2398
- partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
2399
- );
3540
+ async function attachPagedSegment(directory, ref, partitionId, manager, options) {
3541
+ if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
3542
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
3543
+ requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
3544
+ requireAgreedSchema(options.schema, snapshot.schema, ref.key);
3545
+ const segment = createPagedFrozenSegment(snapshot);
3546
+ removeFromPartition(manager, partitionId, snapshot.tombstones);
3547
+ options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
3548
+ if (snapshot.documentCount > 0) {
3549
+ manager.attachFrozenSegment(partitionId, segment);
3550
+ return segment;
2400
3551
  }
2401
- return { targets, contents, plans };
3552
+ segment.paged?.close();
3553
+ return null;
2402
3554
  }
2403
- async function buildArchiveFromMemory(input) {
2404
- const captured = input.partitionId === null ? await whileGraphsStayComplete(input.vectorIndexes, () => capture(input)) : capture(input);
2405
- const store = memoryCheckpointStore();
2406
- const written = await writeCheckpointSegments({
2407
- store,
2408
- metadata: input.metadata,
2409
- targets: captured.targets,
2410
- compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
2411
- prior: null,
2412
- wholePartitions: captured.contents,
2413
- keepsWriteRecords: input.writeRecords != null
2414
- });
2415
- const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
2416
- const refs = vectors.refs;
2417
- await commitCheckpointManifest(store, input.metadata, written, refs);
2418
- const manifestBytes = store.files.get(manifestKey(input.indexName));
2419
- const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName, input.partitionId);
2420
- if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
2421
- throw new NarsilError(
2422
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2423
- `The archive of index "${input.indexName}" holds no manifest`
2424
- );
3555
+ function documentsHeldBefore(manager, partitionId) {
3556
+ const held = manager.partitionAt(partitionId);
3557
+ return held === void 0 ? [] : [...held.docIds()];
3558
+ }
3559
+ function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
3560
+ const absent = heldBefore.filter(
3561
+ (docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
3562
+ );
3563
+ if (absent.length === 0) return;
3564
+ manager.beginBatchRemove();
3565
+ try {
3566
+ for (const docId of absent) manager.remove(docId);
3567
+ } finally {
3568
+ manager.endBatchRemove();
2425
3569
  }
2426
- return {
2427
- manifestBytes,
2428
- manifest: decoded.manifest,
2429
- readFile: async (ref) => {
2430
- const bytes = store.files.get(ref.key);
2431
- if (bytes === void 0) {
2432
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The archive file "${ref.key}" is missing`, {
2433
- key: ref.key
2434
- });
2435
- }
2436
- return bytes;
2437
- },
2438
- readRange: async (ref, offset, length) => {
2439
- const bytes = store.files.get(ref.key);
2440
- if (bytes === void 0 || offset + length > bytes.byteLength) {
2441
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive file "${ref.key}" is missing`, {
2442
- key: ref.key
2443
- });
3570
+ }
3571
+ async function loadPartitionSegments(store, partition, manager, options) {
3572
+ const heldBefore = documentsHeldBefore(manager, partition.partitionId);
3573
+ const loaded = await attachPartitionSegments(store, partition, manager, options);
3574
+ removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
3575
+ }
3576
+ async function attachPartitionSegments(store, partition, manager, options) {
3577
+ const loaded = [];
3578
+ const { directory } = store;
3579
+ for (const ref of partition.segments) {
3580
+ if (manager.config.textStorage === "disk" && directory !== void 0) {
3581
+ if (await isPagedSegmentFile(directory, ref.key)) {
3582
+ const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
3583
+ if (paged !== null) loaded.push(paged);
3584
+ await yieldToEventLoop2();
3585
+ continue;
2444
3586
  }
2445
- return bytes.subarray(offset, offset + length);
2446
- },
2447
- release: () => store.files.clear()
2448
- };
3587
+ }
3588
+ const contents = await readSegmentFile(store, ref);
3589
+ requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
3590
+ requireAgreedSchema(options.schema, contents.schema, ref.key);
3591
+ if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
3592
+ const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
3593
+ removeWhatTheSegmentReplaces(manager, segment, contents);
3594
+ options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
3595
+ if (contents.arrays.documentCount > 0) {
3596
+ manager.attachFrozenSegment(partition.partitionId, segment);
3597
+ loaded.push(segment);
3598
+ }
3599
+ await yieldToEventLoop2();
3600
+ }
3601
+ return loaded;
2449
3602
  }
2450
3603
 
2451
- // src/persistence/archive/write.ts
2452
- function entryChunks(name, bytes) {
2453
- return [...tarEntryHeaders(name, bytes.byteLength), bytes, tarPadding(bytes.byteLength)].filter(
2454
- (chunk) => chunk.byteLength > 0
3604
+ // src/persistence/durability/segment/load.ts
3605
+ async function readSegmentManifest(store, indexName) {
3606
+ const bytes = await store.read(manifestKey(indexName));
3607
+ if (bytes === null) return null;
3608
+ return decodeSegmentManifest(bytes, indexName);
3609
+ }
3610
+ async function readCurrentManifest(store, indexName) {
3611
+ const decoded = await readSegmentManifest(store, indexName);
3612
+ if (decoded === null) return null;
3613
+ if (decoded.kind === "current") return decoded.manifest;
3614
+ throw new NarsilError(
3615
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
3616
+ `Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
3617
+ { indexName }
2455
3618
  );
2456
3619
  }
2457
- async function* fileEntryChunks(source, name, ref) {
2458
- yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
2459
- for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
2460
- yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
2461
- }
2462
- const padding = tarPadding(ref.bytes);
2463
- if (padding.byteLength > 0) yield padding;
3620
+ function growPartitionsTo(manager, highestPartitionId) {
3621
+ manager.growToSavedPartitionCount(highestPartitionId + 1);
2464
3622
  }
2465
- async function* archiveChunks(source) {
2466
- yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
2467
- yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
2468
- const prefix = `${source.sourceIndexName}/`;
2469
- for (const ref of manifestFileRefs(source.manifest)) {
2470
- yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
3623
+ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
3624
+ const highestPartitionId = highestPartitionIdOf(manifest.partitions);
3625
+ requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
3626
+ growPartitionsTo(deps.manager, highestPartitionId);
3627
+ const { directory } = store;
3628
+ const opensVectorsOnDemand = manifest.vectors.some(
3629
+ (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
3630
+ );
3631
+ if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
3632
+ await startReadingOpeningSectionsAhead(directory, manifest);
2471
3633
  }
2472
- if (source.trailingEntries !== void 0) {
2473
- for await (const entry of source.trailingEntries()) yield* entryChunks(entry.name, entry.bytes);
3634
+ for (const partition of manifest.partitions) {
3635
+ await loadPartitionSegments(store, partition, deps.manager, {
3636
+ schema: manifest.schema,
3637
+ decodesEveryDocument,
3638
+ writeRecords: deps.writeRecords ?? null
3639
+ });
2474
3640
  }
2475
- yield tarEnd();
3641
+ for (const vector of manifest.vectors) {
3642
+ const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
3643
+ if (vectorIndex === void 0) continue;
3644
+ await loadVectorField(
3645
+ store,
3646
+ vector,
3647
+ vectorIndex,
3648
+ decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
3649
+ );
3650
+ }
3651
+ return manifest.checkpoint;
2476
3652
  }
2477
- function archiveStream(source) {
2478
- const chunks = archiveChunks(source);
2479
- let released = false;
2480
- const release = () => {
2481
- if (released) return;
2482
- released = true;
2483
- source.release();
2484
- };
2485
- return new ReadableStream({
2486
- async pull(controller) {
2487
- try {
2488
- const step = await chunks.next();
2489
- if (step.done) {
2490
- release();
2491
- controller.close();
2492
- return;
2493
- }
2494
- controller.enqueue(step.value);
2495
- } catch (error) {
2496
- release();
2497
- controller.error(error);
2498
- }
2499
- },
2500
- async cancel() {
2501
- release();
2502
- await chunks.return(void 0);
2503
- }
2504
- });
3653
+ async function reclaimUnlistedKeys(store, indexName, referenced) {
3654
+ for (const key of await store.list(segmentsPrefix(indexName))) {
3655
+ if (!referenced.has(key)) await store.remove(key);
3656
+ }
2505
3657
  }
2506
3658
 
2507
3659
  // src/serialization/stored-vector-values.ts
@@ -2679,38 +3831,71 @@ function deserializePayloadV1(data) {
2679
3831
  function invalid(message, details) {
2680
3832
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The device sync state in the index metadata ${message}`, details);
2681
3833
  }
3834
+ function invalidChanges(message, details) {
3835
+ throw new NarsilError(
3836
+ ErrorCodes.CONFIG_INVALID,
3837
+ `The change capture setting in the index metadata ${message}`,
3838
+ details
3839
+ );
3840
+ }
2682
3841
  function isPositiveSafeInteger(value) {
2683
3842
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
2684
3843
  }
3844
+ function historyLimitsToWire(setting) {
3845
+ return {
3846
+ ...setting.historyMaxAgeMs !== void 0 ? { history_max_age_ms: setting.historyMaxAgeMs } : {},
3847
+ ...setting.historyMaxBytes !== void 0 ? { history_max_bytes: setting.historyMaxBytes } : {}
3848
+ };
3849
+ }
2685
3850
  function syncMetadataToWire(meta) {
2686
3851
  const wire = {};
2687
3852
  if (meta.sync !== void 0) {
2688
- wire.sync = {
2689
- enabled: meta.sync.enabled,
2690
- ...meta.sync.historyMaxAgeMs !== void 0 ? { history_max_age_ms: meta.sync.historyMaxAgeMs } : {},
2691
- ...meta.sync.historyMaxBytes !== void 0 ? { history_max_bytes: meta.sync.historyMaxBytes } : {}
2692
- };
3853
+ wire.sync = { enabled: meta.sync.enabled, ...historyLimitsToWire(meta.sync) };
2693
3854
  }
2694
3855
  if (meta.syncActive !== void 0) wire.sync_active = meta.syncActive;
2695
3856
  if (meta.syncEpoch !== void 0) wire.sync_epoch = meta.syncEpoch;
3857
+ if (meta.changes !== void 0) {
3858
+ wire.changes = {
3859
+ enabled: meta.changes.enabled,
3860
+ ...meta.changes.previousDocument !== void 0 ? { previous_document: meta.changes.previousDocument } : {},
3861
+ ...historyLimitsToWire(meta.changes)
3862
+ };
3863
+ }
2696
3864
  return wire;
2697
3865
  }
2698
- function storedSyncSetting(raw) {
2699
- if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
2700
- const { enabled, history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
2701
- if (typeof enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
3866
+ function historyLimitsFromWire(raw, fail) {
3867
+ const { history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
2702
3868
  if (maxAge !== void 0 && !isPositiveSafeInteger(maxAge)) {
2703
- invalid("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
3869
+ fail("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
2704
3870
  }
2705
3871
  if (maxBytes !== void 0 && !isPositiveSafeInteger(maxBytes)) {
2706
- invalid("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
3872
+ fail("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
2707
3873
  }
2708
3874
  return {
2709
- enabled,
2710
3875
  ...maxAge !== void 0 ? { historyMaxAgeMs: maxAge } : {},
2711
3876
  ...maxBytes !== void 0 ? { historyMaxBytes: maxBytes } : {}
2712
3877
  };
2713
3878
  }
3879
+ function storedSyncSetting(raw) {
3880
+ if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
3881
+ const fields = raw;
3882
+ if (typeof fields.enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
3883
+ return { enabled: fields.enabled, ...historyLimitsFromWire(fields, invalid) };
3884
+ }
3885
+ function storedChangesSetting(raw) {
3886
+ if (raw === null || typeof raw !== "object") invalidChanges("is not a map", {});
3887
+ const fields = raw;
3888
+ if (typeof fields.enabled !== "boolean") invalidChanges("lacks a boolean enabled", {});
3889
+ const previousDocument = fields.previous_document;
3890
+ if (previousDocument !== void 0 && typeof previousDocument !== "boolean") {
3891
+ invalidChanges("holds a previous_document that is not a boolean", {});
3892
+ }
3893
+ return {
3894
+ enabled: fields.enabled,
3895
+ ...previousDocument !== void 0 ? { previousDocument } : {},
3896
+ ...historyLimitsFromWire(fields, invalidChanges)
3897
+ };
3898
+ }
2714
3899
  function syncMetadataFromWire(raw, meta) {
2715
3900
  if (raw.sync !== void 0 && raw.sync !== null) meta.sync = storedSyncSetting(raw.sync);
2716
3901
  if (raw.sync_active !== void 0 && raw.sync_active !== null) {
@@ -2721,6 +3906,7 @@ function syncMetadataFromWire(raw, meta) {
2721
3906
  if (!isSyncEpoch(raw.sync_epoch)) invalid("holds an epoch that is not 32 lowercase hexadecimal digits", {});
2722
3907
  meta.syncEpoch = raw.sync_epoch;
2723
3908
  }
3909
+ if (raw.changes !== void 0 && raw.changes !== null) meta.changes = storedChangesSetting(raw.changes);
2724
3910
  }
2725
3911
 
2726
3912
  // src/serialization/payload-v1/metadata.ts
@@ -3171,202 +4357,32 @@ function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
3171
4357
  manager.remove(entry.documentId, removedSpellings);
3172
4358
  }
3173
4359
 
3174
- // src/persistence/durability/segment/load-partition.ts
3175
- function requireAgreedSchema(agreed, schema, key) {
3176
- for (const [field, type] of Object.entries(schema)) {
3177
- const existing = agreed[field];
3178
- if (existing !== void 0 && existing !== type) {
3179
- throw new NarsilError(
3180
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
3181
- `The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
3182
- { key, field, existing, found: type }
3183
- );
3184
- }
3185
- }
3186
- }
3187
- function requireListedCounts(ref, documentCount, tombstoneCount) {
3188
- if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
3189
- throw new NarsilError(
3190
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
3191
- `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
3192
- {
3193
- key: ref.key,
3194
- documentCount,
3195
- tombstoneCount,
3196
- listedDocCount: ref.docCount,
3197
- listedTombstoneCount: ref.tombstoneCount
3198
- }
3199
- );
3200
- }
3201
- function removeWhatTheSegmentReplaces(manager, segment, contents) {
3202
- manager.beginBatchRemove();
3203
- try {
3204
- for (const docId of contents.tombstones) {
3205
- if (manager.has(docId)) manager.remove(docId);
3206
- }
3207
- const ids = segment.arrays.idTable;
3208
- for (let ordinal = 0; ordinal < ids.count; ordinal++) {
3209
- const docId = ids.idAt(ordinal);
3210
- if (manager.has(docId)) manager.remove(docId);
3211
- }
3212
- } finally {
3213
- manager.endBatchRemove();
3214
- }
3215
- }
3216
- function yieldToEventLoop2() {
3217
- return new Promise((resolve) => setTimeout(resolve, 0));
3218
- }
3219
- function removeFromPartition(manager, partitionId, docIds) {
3220
- const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
3221
- composite.beginBatch();
3222
- try {
3223
- for (const docId of docIds) {
3224
- if (!composite.has(docId)) continue;
3225
- composite.remove(docId, manager.config.schema, manager.language, insertOptions);
3226
- docPartitionMap.delete(docId);
3227
- }
3228
- } finally {
3229
- composite.endBatch();
3230
- }
3231
- }
3232
- async function attachPagedSegment(directory, ref, partitionId, manager, options) {
3233
- if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
3234
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
3235
- requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
3236
- requireAgreedSchema(options.schema, snapshot.schema, ref.key);
3237
- const segment = createPagedFrozenSegment(snapshot);
3238
- removeFromPartition(manager, partitionId, snapshot.tombstones);
3239
- options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
3240
- if (snapshot.documentCount > 0) {
3241
- manager.attachFrozenSegment(partitionId, segment);
3242
- return segment;
3243
- }
3244
- segment.paged?.close();
3245
- return null;
3246
- }
3247
- function documentsHeldBefore(manager, partitionId) {
3248
- const held = manager.partitionAt(partitionId);
3249
- return held === void 0 ? [] : [...held.docIds()];
3250
- }
3251
- function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
3252
- const absent = heldBefore.filter(
3253
- (docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
3254
- );
3255
- if (absent.length === 0) return;
3256
- manager.beginBatchRemove();
3257
- try {
3258
- for (const docId of absent) manager.remove(docId);
3259
- } finally {
3260
- manager.endBatchRemove();
3261
- }
3262
- }
3263
- async function loadPartitionSegments(store, partition, manager, options) {
3264
- const heldBefore = documentsHeldBefore(manager, partition.partitionId);
3265
- const loaded = await attachPartitionSegments(store, partition, manager, options);
3266
- removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
3267
- }
3268
- async function attachPartitionSegments(store, partition, manager, options) {
3269
- const loaded = [];
3270
- const { directory } = store;
3271
- for (const ref of partition.segments) {
3272
- if (manager.config.textStorage === "disk" && directory !== void 0) {
3273
- if (await isPagedSegmentFile(directory, ref.key)) {
3274
- const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
3275
- if (paged !== null) loaded.push(paged);
3276
- await yieldToEventLoop2();
3277
- continue;
3278
- }
4360
+ // src/workers/detached-index.ts
4361
+ function attachablePolicy(shared) {
4362
+ let attached = false;
4363
+ const policy = {
4364
+ get enabled() {
4365
+ return attached && shared.enabled;
4366
+ },
4367
+ get count() {
4368
+ return shared.count;
4369
+ },
4370
+ get host() {
4371
+ return shared.host;
3279
4372
  }
3280
- const contents = await readSegmentFile(store, ref);
3281
- requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
3282
- requireAgreedSchema(options.schema, contents.schema, ref.key);
3283
- if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
3284
- const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
3285
- removeWhatTheSegmentReplaces(manager, segment, contents);
3286
- options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
3287
- if (contents.arrays.documentCount > 0) {
3288
- manager.attachFrozenSegment(partition.partitionId, segment);
3289
- loaded.push(segment);
4373
+ };
4374
+ return {
4375
+ policy,
4376
+ attach() {
4377
+ attached = true;
3290
4378
  }
3291
- await yieldToEventLoop2();
3292
- }
3293
- return loaded;
3294
- }
3295
-
3296
- // src/persistence/durability/segment/load.ts
3297
- async function readSegmentManifest(store, indexName) {
3298
- const bytes = await store.read(manifestKey(indexName));
3299
- if (bytes === null) return null;
3300
- return decodeSegmentManifest(bytes, indexName);
3301
- }
3302
- async function readCurrentManifest(store, indexName) {
3303
- const decoded = await readSegmentManifest(store, indexName);
3304
- if (decoded === null) return null;
3305
- if (decoded.kind === "current") return decoded.manifest;
3306
- throw new NarsilError(
3307
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
3308
- `Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
3309
- { indexName }
3310
- );
3311
- }
3312
- function growPartitionsTo(manager, highestPartitionId) {
3313
- manager.growToSavedPartitionCount(highestPartitionId + 1);
3314
- }
3315
- async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
3316
- const highestPartitionId = highestPartitionIdOf(manifest.partitions);
3317
- requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
3318
- growPartitionsTo(deps.manager, highestPartitionId);
3319
- const { directory } = store;
3320
- const opensVectorsOnDemand = manifest.vectors.some(
3321
- (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
3322
- );
3323
- if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
3324
- await startReadingOpeningSectionsAhead(directory, manifest);
3325
- }
3326
- for (const partition of manifest.partitions) {
3327
- await loadPartitionSegments(store, partition, deps.manager, {
3328
- schema: manifest.schema,
3329
- decodesEveryDocument,
3330
- writeRecords: deps.writeRecords ?? null
3331
- });
3332
- }
3333
- for (const vector of manifest.vectors) {
3334
- const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
3335
- if (vectorIndex === void 0) continue;
3336
- await loadVectorField(
3337
- store,
3338
- vector,
3339
- vectorIndex,
3340
- decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
3341
- );
3342
- }
3343
- return manifest.checkpoint;
3344
- }
3345
- async function reclaimUnlistedKeys(store, indexName, referenced) {
3346
- for (const key of await store.list(segmentsPrefix(indexName))) {
3347
- if (!referenced.has(key)) await store.remove(key);
3348
- }
4379
+ };
3349
4380
  }
3350
-
3351
- // src/persistence/archive/file-entry.ts
3352
- async function writeArchiveFile(store, key, chunks) {
3353
- const { directory } = store;
3354
- if (directory === void 0) {
3355
- const held = [];
3356
- for await (const chunk of chunks) held.push(chunk);
3357
- await store.write(key, held);
3358
- return;
3359
- }
3360
- const receiving = `${key}.receiving`;
3361
- await directory.remove(receiving);
3362
- const handle = await directory.appendHandle(receiving);
3363
- try {
3364
- for await (const chunk of chunks) await handle.append(chunk);
3365
- await handle.sync();
3366
- } finally {
3367
- await handle.close();
3368
- }
3369
- await directory.move(receiving, key);
4381
+ function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
4382
+ const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
4383
+ const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
4384
+ entry.manager.synonyms = synonyms;
4385
+ return { indexName, manager: entry.manager, entry, attach };
3370
4386
  }
3371
4387
 
3372
4388
  // src/persistence/durability/marker-writer.ts
@@ -3424,7 +4440,7 @@ async function loadMetadata(directory, indexName) {
3424
4440
  return metadata;
3425
4441
  }
3426
4442
  async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
3427
- await directory.remove(nextSegmentKey(indexName, partitionId));
4443
+ await removeUnusedSegmentFiles(directory, indexName, partitionId);
3428
4444
  const read = await readWalSegments(directory, indexName, partitionId);
3429
4445
  if (read === null) {
3430
4446
  return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
@@ -3488,7 +4504,7 @@ async function replayWalUpTo(directory, indexName, partitionId, fromSeqNoExclusi
3488
4504
  return replayDurableEntries(read, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps);
3489
4505
  }
3490
4506
  async function replaySyncedWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
3491
- await directory.remove(nextSegmentKey(indexName, partitionId));
4507
+ await removeUnusedSegmentFiles(directory, indexName, partitionId);
3492
4508
  const read = await readWalSegments(directory, indexName, partitionId);
3493
4509
  if (read === null) {
3494
4510
  return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
@@ -3548,6 +4564,7 @@ async function truncateSegmentTail(directory, key, durableByteLength) {
3548
4564
  }
3549
4565
  function applyEntry2(entry, deps) {
3550
4566
  deps.writeRecords?.replayed(entry);
4567
+ if (entry.operation === "DELETE" || deps.manager.has(entry.documentId)) noteDocumentChurn(entry.indexName, 1);
3551
4568
  if (entry.operation === "DELETE") {
3552
4569
  applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
3553
4570
  return;
@@ -3576,32 +4593,4 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
3576
4593
  }
3577
4594
  }
3578
4595
 
3579
- // src/workers/detached-index.ts
3580
- function attachablePolicy(shared) {
3581
- let attached = false;
3582
- const policy = {
3583
- get enabled() {
3584
- return attached && shared.enabled;
3585
- },
3586
- get count() {
3587
- return shared.count;
3588
- },
3589
- get host() {
3590
- return shared.host;
3591
- }
3592
- };
3593
- return {
3594
- policy,
3595
- attach() {
3596
- attached = true;
3597
- }
3598
- };
3599
- }
3600
- function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
3601
- const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
3602
- const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
3603
- entry.manager.synonyms = synonyms;
3604
- return { indexName, manager: entry.manager, entry, attach };
3605
- }
3606
-
3607
- export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, isRecord2 as isRecord, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
4596
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, changesOf, changesStatusOf, checkpointInTurn, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, documentExportStream, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, followIndexChanges, followSourceChanges, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openFileVectorReads, pagedPartitionLines, patternIndexesFromWire, readCurrentManifest, readDocumentExport, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireChangesSetting, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, updateChangesOf, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };