@delali/narsil 0.3.1-next.157 → 0.3.1-next.161

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (565) hide show
  1. package/dist/adapters/filesystem.mjs +3 -3
  2. package/dist/adapters/gcs.mjs +3 -3
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/adapters/s3.mjs +3 -3
  5. package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
  6. package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
  7. package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
  8. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  9. package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
  10. package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
  11. package/dist/browser/embed-RF2MPGAO.mjs +222 -0
  12. package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
  13. package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
  14. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  15. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  16. package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
  17. package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
  18. package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
  19. package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
  20. package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
  21. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  22. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  23. package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
  24. package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
  25. package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
  26. package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
  27. package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
  28. package/dist/chunk-2JRERUNZ.mjs +217 -0
  29. package/dist/chunk-2RIS76RJ.mjs +212 -0
  30. package/dist/chunk-3B775F3C.mjs +252 -0
  31. package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
  32. package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
  33. package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
  34. package/dist/chunk-6EZW2GQL.mjs +978 -0
  35. package/dist/chunk-75UNINFA.mjs +271 -0
  36. package/dist/chunk-77AUVDQ5.mjs +7 -0
  37. package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
  38. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  39. package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
  40. package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
  41. package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
  42. package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
  43. package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
  44. package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
  45. package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
  46. package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
  47. package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
  48. package/dist/chunk-GXCDWLQI.mjs +193 -0
  49. package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
  50. package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
  51. package/dist/chunk-IB4BYLE3.mjs +127 -0
  52. package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
  53. package/dist/chunk-IH3G6SJO.mjs +40 -0
  54. package/dist/chunk-JV74RKCY.mjs +3041 -0
  55. package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
  56. package/dist/chunk-KLVMK3R5.mjs +47 -0
  57. package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
  58. package/dist/chunk-LF4GB4PO.mjs +281 -0
  59. package/dist/chunk-NC4QLCZC.mjs +59 -0
  60. package/dist/chunk-OOO54OVC.mjs +627 -0
  61. package/dist/chunk-OUWZKNO3.mjs +207 -0
  62. package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
  63. package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
  64. package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
  65. package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
  66. package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
  67. package/dist/chunk-S2POFTLN.mjs +10 -0
  68. package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
  69. package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
  70. package/dist/chunk-SXIRM4MX.mjs +3549 -0
  71. package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
  72. package/dist/chunk-TINW4BSH.mjs +31301 -0
  73. package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
  74. package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
  75. package/dist/chunk-U6B7DUQW.mjs +788 -0
  76. package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
  77. package/dist/chunk-W34VJXY7.mjs +4723 -0
  78. package/dist/chunk-YETKPX5M.mjs +5 -0
  79. package/dist/chunk-ZQ6QY42L.mjs +59 -0
  80. package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
  81. package/dist/client.d.ts +8 -8
  82. package/dist/client.mjs +66 -18
  83. package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
  84. package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
  85. package/dist/core-leaf-KXFGP74A.mjs +7 -0
  86. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  87. package/dist/direct-executor-MKTZBR26.mjs +16 -0
  88. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  89. package/dist/distribution/coordinator/etcd.mjs +7 -4
  90. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  91. package/dist/distribution/coordinator/in-memory.mjs +1 -2
  92. package/dist/distribution/transport/grpc.mjs +7 -2
  93. package/dist/distribution/transport/tcp.mjs +7 -1
  94. package/dist/distribution.d.ts +7 -7
  95. package/dist/distribution.mjs +1199 -1092
  96. package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
  97. package/dist/embed-4H7RTBVL.mjs +221 -0
  98. package/dist/embeddings/openai.mjs +2 -2
  99. package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
  100. package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
  101. package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
  102. package/dist/index.browser.mjs +4 -16
  103. package/dist/index.d.ts +1495 -78
  104. package/dist/index.mjs +14 -50
  105. package/dist/index.react-native.mjs +11 -28
  106. package/dist/invalidation/filesystem.mjs +1 -2
  107. package/dist/languages/albanian.mjs +1 -1
  108. package/dist/languages/amharic.mjs +1 -1
  109. package/dist/languages/arabic.mjs +1 -1
  110. package/dist/languages/armenian.mjs +1 -1
  111. package/dist/languages/azerbaijani.mjs +1 -1
  112. package/dist/languages/bambara.mjs +1 -1
  113. package/dist/languages/basque.mjs +1 -1
  114. package/dist/languages/belarusian.mjs +1 -1
  115. package/dist/languages/bengali.mjs +1 -1
  116. package/dist/languages/bosnian.mjs +1 -1
  117. package/dist/languages/breton.mjs +1 -1
  118. package/dist/languages/bulgarian.mjs +1 -1
  119. package/dist/languages/burmese.mjs +1 -1
  120. package/dist/languages/catalan.mjs +1 -1
  121. package/dist/languages/chinese.mjs +1 -1
  122. package/dist/languages/croatian.mjs +1 -1
  123. package/dist/languages/czech.mjs +1 -1
  124. package/dist/languages/dagbani.mjs +1 -1
  125. package/dist/languages/danish.mjs +1 -1
  126. package/dist/languages/dutch.mjs +1 -1
  127. package/dist/languages/english.mjs +1 -1
  128. package/dist/languages/esperanto.mjs +1 -1
  129. package/dist/languages/estonian.mjs +1 -1
  130. package/dist/languages/ewe.mjs +1 -1
  131. package/dist/languages/faroese.mjs +1 -1
  132. package/dist/languages/fijian.mjs +1 -1
  133. package/dist/languages/finnish.mjs +1 -1
  134. package/dist/languages/french.mjs +1 -1
  135. package/dist/languages/ga.mjs +1 -1
  136. package/dist/languages/galician.mjs +1 -1
  137. package/dist/languages/georgian.mjs +1 -1
  138. package/dist/languages/german.mjs +1 -1
  139. package/dist/languages/greek.mjs +1 -1
  140. package/dist/languages/guarani.mjs +1 -1
  141. package/dist/languages/gujarati.mjs +1 -1
  142. package/dist/languages/haitian-creole.mjs +1 -1
  143. package/dist/languages/hausa.mjs +1 -1
  144. package/dist/languages/hawaiian.mjs +1 -1
  145. package/dist/languages/hebrew.mjs +1 -1
  146. package/dist/languages/hindi.mjs +1 -1
  147. package/dist/languages/hungarian.mjs +1 -1
  148. package/dist/languages/icelandic.mjs +1 -1
  149. package/dist/languages/igbo.mjs +1 -1
  150. package/dist/languages/indonesian.mjs +1 -1
  151. package/dist/languages/irish.mjs +1 -1
  152. package/dist/languages/italian.mjs +1 -1
  153. package/dist/languages/japanese.mjs +1 -1
  154. package/dist/languages/kannada.mjs +1 -1
  155. package/dist/languages/kazakh.mjs +1 -1
  156. package/dist/languages/khmer.mjs +1 -1
  157. package/dist/languages/kinyarwanda.mjs +1 -1
  158. package/dist/languages/kirundi.mjs +1 -1
  159. package/dist/languages/korean.mjs +1 -1
  160. package/dist/languages/kurmanji.mjs +1 -1
  161. package/dist/languages/kyrgyz.mjs +1 -1
  162. package/dist/languages/lao.mjs +1 -1
  163. package/dist/languages/latin.mjs +1 -1
  164. package/dist/languages/latvian.mjs +1 -1
  165. package/dist/languages/lingala.mjs +1 -1
  166. package/dist/languages/lithuanian.mjs +1 -1
  167. package/dist/languages/luxembourgish.mjs +1 -1
  168. package/dist/languages/macedonian.mjs +1 -1
  169. package/dist/languages/malagasy.mjs +1 -1
  170. package/dist/languages/malay.mjs +1 -1
  171. package/dist/languages/malayalam.mjs +1 -1
  172. package/dist/languages/maltese.mjs +1 -1
  173. package/dist/languages/maori.mjs +1 -1
  174. package/dist/languages/marathi.mjs +1 -1
  175. package/dist/languages/nepali.mjs +1 -1
  176. package/dist/languages/norwegian.mjs +1 -1
  177. package/dist/languages/oromo.mjs +1 -1
  178. package/dist/languages/persian.mjs +1 -1
  179. package/dist/languages/polish.mjs +1 -1
  180. package/dist/languages/portuguese.mjs +1 -1
  181. package/dist/languages/punjabi.mjs +1 -1
  182. package/dist/languages/romanian.mjs +1 -1
  183. package/dist/languages/russian.mjs +1 -1
  184. package/dist/languages/samoan.mjs +1 -1
  185. package/dist/languages/sanskrit.mjs +1 -1
  186. package/dist/languages/scottish-gaelic.mjs +1 -1
  187. package/dist/languages/serbian.mjs +1 -1
  188. package/dist/languages/shona.mjs +1 -1
  189. package/dist/languages/sinhala.mjs +1 -1
  190. package/dist/languages/slovak.mjs +1 -1
  191. package/dist/languages/slovenian.mjs +1 -1
  192. package/dist/languages/sorani.mjs +1 -1
  193. package/dist/languages/spanish.mjs +1 -1
  194. package/dist/languages/swahili.mjs +1 -1
  195. package/dist/languages/swedish.mjs +1 -1
  196. package/dist/languages/tagalog.mjs +1 -1
  197. package/dist/languages/tamil.mjs +1 -1
  198. package/dist/languages/tatar.mjs +1 -1
  199. package/dist/languages/telugu.mjs +1 -1
  200. package/dist/languages/thai.mjs +1 -1
  201. package/dist/languages/tibetan.mjs +1 -1
  202. package/dist/languages/tigrinya.mjs +1 -1
  203. package/dist/languages/tongan.mjs +1 -1
  204. package/dist/languages/turkish.mjs +1 -1
  205. package/dist/languages/twi.mjs +1 -1
  206. package/dist/languages/ukrainian.mjs +1 -1
  207. package/dist/languages/urdu.mjs +1 -1
  208. package/dist/languages/vietnamese.mjs +1 -1
  209. package/dist/languages/welsh.mjs +1 -1
  210. package/dist/languages/wolof.mjs +1 -1
  211. package/dist/languages/xhosa.mjs +1 -1
  212. package/dist/languages/yoruba.mjs +1 -1
  213. package/dist/languages/zulu.mjs +1 -1
  214. package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
  215. package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
  216. package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
  217. package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
  218. package/dist/optional-writes-DDFHYUOH.mjs +218 -0
  219. package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
  220. package/dist/partial-words.browser.mjs +1649 -0
  221. package/dist/partial-words.mjs +15 -14
  222. package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
  223. package/dist/pattern.mjs +6 -8
  224. package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
  225. package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
  226. package/dist/phrase-module-XK3T44VU.mjs +634 -0
  227. package/dist/query-actions-OMXJEC2J.mjs +118 -0
  228. package/dist/query-suggestions/merge-worker.mjs +18 -54
  229. package/dist/query-suggestions.browser.mjs +26 -227
  230. package/dist/query-suggestions.d.ts +2 -2
  231. package/dist/query-suggestions.mjs +16 -67
  232. package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
  233. package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
  234. package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
  235. package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
  236. package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
  237. package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
  238. package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
  239. package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
  240. package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
  241. package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
  242. package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
  243. package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
  244. package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
  245. package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
  246. package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
  247. package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
  248. package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
  249. package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
  250. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  251. package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
  252. package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
  253. package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
  254. package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
  255. package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
  256. package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
  257. package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
  258. package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
  259. package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
  260. package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
  261. package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
  262. package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
  263. package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
  264. package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
  265. package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
  266. package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
  267. package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
  268. package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
  269. package/dist/react-native/partial-words.mjs +954 -35
  270. package/dist/react-native/pattern.mjs +238 -111
  271. package/dist/react-native/query-suggestions.mjs +33 -239
  272. package/dist/react-native/sync.mjs +95 -52
  273. package/dist/react-native/synonyms.mjs +256 -112
  274. package/dist/react.d.ts +9 -9
  275. package/dist/react.mjs +6 -0
  276. package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
  277. package/dist/rerankers/http.mjs +2 -2
  278. package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
  279. package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
  280. package/dist/runtime-4GKUIAKV.mjs +595 -0
  281. package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
  282. package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
  283. package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
  284. package/dist/server.d.ts +33 -13
  285. package/dist/server.mjs +81 -100
  286. package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
  287. package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
  288. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  289. package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
  290. package/dist/sync/filesystem.d.ts +3 -3
  291. package/dist/sync/filesystem.mjs +27 -11
  292. package/dist/sync/indexeddb.browser.mjs +287 -0
  293. package/dist/sync/indexeddb.d.ts +3 -3
  294. package/dist/sync/indexeddb.mjs +7 -2
  295. package/dist/sync.browser.mjs +88 -72
  296. package/dist/sync.d.ts +4 -4
  297. package/dist/sync.mjs +89 -59
  298. package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
  299. package/dist/synonyms.mjs +15 -11
  300. package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
  301. package/dist/thread-SNILJXX3.mjs +30 -0
  302. package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
  303. package/dist/vector/search-worker.d.ts +7 -1
  304. package/dist/vector/search-worker.mjs +9 -13
  305. package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
  306. package/dist/workers/entry.mjs +60 -30
  307. package/openapi.json +66 -50
  308. package/package.json +233 -8
  309. package/dist/analysis-change-CKEMM2MC.mjs +0 -55
  310. package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
  311. package/dist/archive-history-E4R73RWD.mjs +0 -29
  312. package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
  313. package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
  314. package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
  315. package/dist/browser/chunk-25MTVDYD.mjs +0 -432
  316. package/dist/browser/chunk-3I27KXQD.mjs +0 -34
  317. package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
  318. package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
  319. package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
  320. package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
  321. package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
  322. package/dist/browser/chunk-AI5UI46D.mjs +0 -12
  323. package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
  324. package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
  325. package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
  326. package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
  327. package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
  328. package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
  329. package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
  330. package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
  331. package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
  332. package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
  333. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  334. package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
  335. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  336. package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
  337. package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
  338. package/dist/browser/chunk-MY2XKH22.mjs +0 -40
  339. package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
  340. package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
  341. package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
  342. package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
  343. package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
  344. package/dist/browser/chunk-TK63WFXK.mjs +0 -90
  345. package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
  346. package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
  347. package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
  348. package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
  349. package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
  350. package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
  351. package/dist/browser/chunk-V43N6TZO.mjs +0 -857
  352. package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
  353. package/dist/browser/chunk-WN464PTL.mjs +0 -375
  354. package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
  355. package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
  356. package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
  357. package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
  358. package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
  359. package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
  360. package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
  361. package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
  362. package/dist/browser/import-5ESVMFDJ.mjs +0 -20
  363. package/dist/browser/methods-KTQCSIN4.mjs +0 -30
  364. package/dist/browser/module-NDSQLPMQ.mjs +0 -25
  365. package/dist/browser/operations-OJ3BXS55.mjs +0 -5
  366. package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
  367. package/dist/browser/pool-3JLET5RO.mjs +0 -297
  368. package/dist/browser/release-HYXPVJVS.mjs +0 -32
  369. package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
  370. package/dist/browser/retention-TY3L3BFN.mjs +0 -211
  371. package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
  372. package/dist/browser/start-2UZTOBG5.mjs +0 -26
  373. package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
  374. package/dist/browser/write-ULXAN5B4.mjs +0 -196
  375. package/dist/chunk-2NE7VCYE.mjs +0 -212
  376. package/dist/chunk-2ROUFMQX.mjs +0 -543
  377. package/dist/chunk-37CD3OKS.mjs +0 -87
  378. package/dist/chunk-3MXV3VHR.mjs +0 -46
  379. package/dist/chunk-3OH67G7R.mjs +0 -214
  380. package/dist/chunk-4B7JIGQD.mjs +0 -106
  381. package/dist/chunk-5YNRKI2C.mjs +0 -81
  382. package/dist/chunk-6HTGOXIK.mjs +0 -875
  383. package/dist/chunk-6OEGXLBX.mjs +0 -57
  384. package/dist/chunk-6RC4K46J.mjs +0 -86
  385. package/dist/chunk-7GFE3PEY.mjs +0 -40
  386. package/dist/chunk-7GXJUJ24.mjs +0 -49
  387. package/dist/chunk-7V5IUP2T.mjs +0 -63
  388. package/dist/chunk-A4L7YTKX.mjs +0 -458
  389. package/dist/chunk-BEZH7VEH.mjs +0 -23
  390. package/dist/chunk-BNL5OPRV.mjs +0 -634
  391. package/dist/chunk-BTPGQC6Z.mjs +0 -27
  392. package/dist/chunk-C7XQYG6U.mjs +0 -50
  393. package/dist/chunk-CQZXZ75V.mjs +0 -562
  394. package/dist/chunk-CWS55TEX.mjs +0 -242
  395. package/dist/chunk-CZQ33DLW.mjs +0 -78
  396. package/dist/chunk-EEBCDP36.mjs +0 -32
  397. package/dist/chunk-FB6463IN.mjs +0 -1499
  398. package/dist/chunk-FCJ553HS.mjs +0 -655
  399. package/dist/chunk-FEZVDOWX.mjs +0 -6996
  400. package/dist/chunk-FOCV4CBZ.mjs +0 -87
  401. package/dist/chunk-FQBFZAWG.mjs +0 -27
  402. package/dist/chunk-G6HRSGOU.mjs +0 -74
  403. package/dist/chunk-GES7U2PM.mjs +0 -68
  404. package/dist/chunk-GH4WQ22V.mjs +0 -34
  405. package/dist/chunk-GPWMUPCA.mjs +0 -1126
  406. package/dist/chunk-GQPTDQCL.mjs +0 -54
  407. package/dist/chunk-HJSN7UYG.mjs +0 -10
  408. package/dist/chunk-I54T6CQP.mjs +0 -117
  409. package/dist/chunk-IPN7S4KX.mjs +0 -130
  410. package/dist/chunk-J4IJTTEX.mjs +0 -226
  411. package/dist/chunk-JSV6OQ4C.mjs +0 -74
  412. package/dist/chunk-K4ZIGL2O.mjs +0 -290
  413. package/dist/chunk-KAKQ74LS.mjs +0 -53
  414. package/dist/chunk-KCQYFO4K.mjs +0 -62
  415. package/dist/chunk-KHKIRS5D.mjs +0 -113
  416. package/dist/chunk-KOKM4PQN.mjs +0 -52
  417. package/dist/chunk-KP7TYHWT.mjs +0 -92
  418. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  419. package/dist/chunk-L5ORED7V.mjs +0 -25
  420. package/dist/chunk-LDNCLRZU.mjs +0 -218
  421. package/dist/chunk-LHKOSI2J.mjs +0 -58
  422. package/dist/chunk-LQG5G2LY.mjs +0 -39
  423. package/dist/chunk-MATMVH65.mjs +0 -9
  424. package/dist/chunk-MP7ABPTD.mjs +0 -189
  425. package/dist/chunk-MW6BR4QP.mjs +0 -316
  426. package/dist/chunk-MWD5VPN7.mjs +0 -21
  427. package/dist/chunk-ODYKO5TO.mjs +0 -101
  428. package/dist/chunk-OG7UGJ6F.mjs +0 -63
  429. package/dist/chunk-ORN5BLL7.mjs +0 -11
  430. package/dist/chunk-OYABFKTW.mjs +0 -1974
  431. package/dist/chunk-P5LANSH6.mjs +0 -121
  432. package/dist/chunk-PELHVTO7.mjs +0 -24
  433. package/dist/chunk-PIH5SJ4N.mjs +0 -7481
  434. package/dist/chunk-PSZI4O24.mjs +0 -253
  435. package/dist/chunk-PV2CG7EJ.mjs +0 -886
  436. package/dist/chunk-PWGYQXML.mjs +0 -470
  437. package/dist/chunk-Q2UL2TPC.mjs +0 -21
  438. package/dist/chunk-QHPYS427.mjs +0 -66
  439. package/dist/chunk-QJVOLKU6.mjs +0 -111
  440. package/dist/chunk-QLDABJZM.mjs +0 -20
  441. package/dist/chunk-QYRNLLLI.mjs +0 -99
  442. package/dist/chunk-RCX4WNLC.mjs +0 -151
  443. package/dist/chunk-RFBLQLLZ.mjs +0 -16
  444. package/dist/chunk-RIHIVFIQ.mjs +0 -240
  445. package/dist/chunk-RLOES5SX.mjs +0 -31
  446. package/dist/chunk-RWURUDSP.mjs +0 -2639
  447. package/dist/chunk-RZWWB2XR.mjs +0 -96
  448. package/dist/chunk-S5LN746F.mjs +0 -10
  449. package/dist/chunk-SOI6LTHM.mjs +0 -216
  450. package/dist/chunk-SQJWAIF4.mjs +0 -12
  451. package/dist/chunk-TDIK6NTV.mjs +0 -10
  452. package/dist/chunk-TQXHPFWT.mjs +0 -177
  453. package/dist/chunk-U55XJEIG.mjs +0 -18
  454. package/dist/chunk-UD5HIS4U.mjs +0 -382
  455. package/dist/chunk-UHICS3JI.mjs +0 -83
  456. package/dist/chunk-UJDJB3NE.mjs +0 -913
  457. package/dist/chunk-VD7J552I.mjs +0 -28
  458. package/dist/chunk-VSLSGASN.mjs +0 -30
  459. package/dist/chunk-VZ2ONHEE.mjs +0 -3715
  460. package/dist/chunk-W7OSSKH5.mjs +0 -1427
  461. package/dist/chunk-WZ7YV54N.mjs +0 -152
  462. package/dist/chunk-X5CV2E2B.mjs +0 -1
  463. package/dist/chunk-Y4WPNHMC.mjs +0 -50
  464. package/dist/chunk-YEKL4JBK.mjs +0 -96
  465. package/dist/chunk-ZXNQEW43.mjs +0 -18
  466. package/dist/chunk-ZYAY26KG.mjs +0 -29
  467. package/dist/direct-executor-V5DRIECI.mjs +0 -34
  468. package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
  469. package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
  470. package/dist/gzip-F4HMIEEQ.mjs +0 -1
  471. package/dist/import-GPKLTSKA.mjs +0 -56
  472. package/dist/methods-7IOINERF.mjs +0 -30
  473. package/dist/module-JYHFTVPS.mjs +0 -63
  474. package/dist/operations-G5EP6JPX.mjs +0 -5
  475. package/dist/phrase-module-XW5MKWP5.mjs +0 -330
  476. package/dist/pool-Y4A46T36.mjs +0 -298
  477. package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
  478. package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
  479. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
  480. package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
  481. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
  482. package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
  483. package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
  484. package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
  485. package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
  486. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
  487. package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
  488. package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
  489. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
  490. package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
  491. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
  492. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
  493. package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
  494. package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
  495. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
  496. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
  497. package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
  498. package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
  499. package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
  500. package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
  501. package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
  502. package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
  503. package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
  504. package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
  505. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
  506. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
  507. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
  508. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
  509. package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
  510. package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
  511. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
  512. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
  513. package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
  514. package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
  515. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
  516. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
  517. package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
  518. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
  519. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
  520. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
  521. package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
  522. package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
  523. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
  524. package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
  525. package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
  526. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
  527. package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
  528. package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
  529. package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
  530. package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
  531. package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
  532. package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
  533. package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
  534. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
  535. package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
  536. package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
  537. package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
  538. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
  539. package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
  540. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
  541. package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
  542. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
  543. package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
  544. package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
  545. package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
  546. package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
  547. package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
  548. package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
  549. package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
  550. package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
  551. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
  552. package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
  553. package/dist/release-SMR4WJD4.mjs +0 -32
  554. package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
  555. package/dist/restore-queue-LZH7LMLR.mjs +0 -55
  556. package/dist/retention-3K7ARGAG.mjs +0 -66
  557. package/dist/runtime-COIUORCN.mjs +0 -177
  558. package/dist/snapshot-O7ICP7NJ.mjs +0 -783
  559. package/dist/start-S4PFP6RS.mjs +0 -66
  560. package/dist/sync-state-ZWHLAFLE.mjs +0 -15
  561. package/dist/thread-YLJ3ZPQ3.mjs +0 -50
  562. package/dist/write-2OBJGLFS.mjs +0 -233
  563. /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
  564. /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
  565. /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
@@ -1,2562 +0,0 @@
1
- import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor } from './chunk-HJBJBH26.mjs';
2
- import { gunzip, gzip } from './chunk-CKNRFDTZ.mjs';
3
- import { readListedFile, requireListedChecksum, writeSegmentsOf, readSegmentFile, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-5MSX5FRT.mjs';
4
- import { DEFAULT_COMPACTION_THRESHOLD, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-F753DA54.mjs';
5
- import { decodeSegmentManifest } from './chunk-BITYFZSZ.mjs';
6
- import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead } from './chunk-DVDX4SDR.mjs';
7
- import { restoreVectorFields } from './chunk-OXM66LO6.mjs';
8
- import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, vectorFileKey, MAX_SEGMENTS_PER_PARTITION, graphPartKey } from './chunk-6OFDUYVX.mjs';
9
- import { isSyncEpoch } from './chunk-JWUCMVRK.mjs';
10
- import { storedStateToWire, storedStateFromWire } from './chunk-KSBYEMCB.mjs';
11
- import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, isCompositePartition, packChecksummedEnvelope, reconstructSchemaFromMetadata, engineVersionParts, packEnvelopeWithChecksum, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeEveryStoredDocument, shareSegmentArrays, decodeMapWithoutFields, prepareDocumentVectors } from './chunk-OCSLSH27.mjs';
12
- import { crc32, readHeader, HEADER_SIZE, writeHeader, openChecksummedEnvelope, checksumOfChunks } from './chunk-YH7QY657.mjs';
13
- import { flattenSchema, extractVectorFieldsFromSchema, requireValidFieldTypes, partialFieldsOf, isQuantizationMode, isStorageMode } from './chunk-4COWCEY7.mjs';
14
- import { VALID_HNSW_METRICS, MAX_FILE_BYTES } from './chunk-6VBXNFLE.mjs';
15
- import { VECTOR_PATTERN } from './chunk-C5G6TJ7A.mjs';
16
- import { getLanguage, resolveIndexAnalysis } from './chunk-MJDQU5MW.mjs';
17
- import { compareCodePoints } from './chunk-EJ33LCHV.mjs';
18
- import { NarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
19
- import { decode, encode } from '@msgpack/msgpack';
20
-
21
- // src/engine/durability-metadata.ts
22
- var VECTOR_TYPE = /^vector\[(\d+)\]$/;
23
- function vectorFieldsOf(schema, promotion) {
24
- const fields = {};
25
- for (const [fieldPath, type] of Object.entries(schema)) {
26
- const match = VECTOR_TYPE.exec(type);
27
- if (match === null) continue;
28
- const dimension = Number(match[1]);
29
- fields[fieldPath] = {
30
- dimension,
31
- metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
32
- quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
33
- };
34
- }
35
- return Object.keys(fields).length === 0 ? null : fields;
36
- }
37
- function metadataFromConfig(indexName, config, partitionCount, documentCount) {
38
- const metadata = {
39
- indexName,
40
- schema: config.schema,
41
- language: config.language,
42
- partitionCount: partitionCount ?? config.partitionCount,
43
- bm25Params: {
44
- k1: config.k1,
45
- b: config.b,
46
- ...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
47
- },
48
- createdAt: Date.now(),
49
- engineVersion: VERSION
50
- };
51
- metadata.documentCount = documentCount ?? config.documentCount;
52
- const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
53
- if (vectorFields !== null) {
54
- metadata.vectorFields = vectorFields;
55
- }
56
- if (config.embedding) {
57
- metadata.embedding = config.embedding;
58
- }
59
- if (config.surfaceForms !== void 0) {
60
- metadata.surfaceForms = config.surfaceForms;
61
- }
62
- if (config.analysisRevision !== void 0) {
63
- metadata.analysisRevision = config.analysisRevision;
64
- }
65
- if (config.arrayPositionGap !== void 0) {
66
- metadata.arrayPositionGap = config.arrayPositionGap;
67
- }
68
- if (config.tokenizer !== void 0) {
69
- metadata.tokenizer = config.tokenizer;
70
- }
71
- if (config.stopWords !== void 0) {
72
- metadata.stopWords = config.stopWords;
73
- }
74
- if (config.stopWordList !== void 0) {
75
- metadata.stopWordList = config.stopWordList;
76
- }
77
- if (config.analysisGeneration !== void 0) {
78
- metadata.analysisGeneration = config.analysisGeneration;
79
- }
80
- if (config.pendingAnalysis !== void 0) {
81
- metadata.pendingAnalysis = config.pendingAnalysis;
82
- }
83
- if (config.partitionLimits !== void 0) {
84
- metadata.partitionLimits = config.partitionLimits;
85
- }
86
- if (config.defaultScoring !== void 0) {
87
- metadata.defaultScoring = config.defaultScoring;
88
- }
89
- if (config.trackPositions !== void 0) {
90
- metadata.trackPositions = config.trackPositions;
91
- }
92
- if (config.textStorage !== void 0) {
93
- metadata.textStorage = config.textStorage;
94
- }
95
- if (config.strict !== void 0) {
96
- metadata.strict = config.strict;
97
- }
98
- if (config.required !== void 0) {
99
- metadata.required = config.required;
100
- }
101
- if (config.vectorPromotion !== void 0) {
102
- metadata.vectorPromotion = config.vectorPromotion;
103
- }
104
- if (config.patternValueLimit !== void 0) {
105
- metadata.patternValueLimit = config.patternValueLimit;
106
- }
107
- if (config.indexUuid !== void 0) {
108
- metadata.indexUuid = config.indexUuid;
109
- }
110
- if (config.synonyms !== void 0) {
111
- metadata.synonyms = config.synonyms;
112
- }
113
- if (config.heldPartitions !== void 0) {
114
- metadata.heldPartitions = config.heldPartitions;
115
- }
116
- if (config.sync !== void 0) metadata.sync = config.sync;
117
- if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
118
- if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
119
- if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
120
- return metadata;
121
- }
122
- function metadataConfigOf(entry) {
123
- const embedding = entry.config.embedding ? {
124
- fields: entry.config.embedding.fields,
125
- ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
126
- } : void 0;
127
- return {
128
- ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
129
- ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
130
- documentCount: entry.documentCount,
131
- partitionCount: entry.partitionCount,
132
- schema: flattenSchema(entry.config.schema),
133
- language: entry.language.name,
134
- k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
135
- b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
136
- ...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
137
- ...embedding !== void 0 ? { embedding } : {},
138
- surfaceForms: entry.config.surfaceForms !== false,
139
- ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
140
- ...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
141
- ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
142
- ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
143
- ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
144
- ...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
145
- ...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
146
- ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
147
- ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
148
- ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
149
- ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
150
- ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
151
- ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
152
- ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
153
- ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
154
- ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
155
- ...syncMetadataOf(entry),
156
- ...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
157
- };
158
- }
159
- function syncMetadataOf(entry) {
160
- const sync = entry.sync;
161
- if (sync === null) return {};
162
- return {
163
- sync: {
164
- enabled: true,
165
- ...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
166
- ...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
167
- },
168
- syncActive: sync.active,
169
- syncEpoch: sync.epoch
170
- };
171
- }
172
- function durableMetadataConfigOf(entry) {
173
- const terms = metadataConfigOf(entry);
174
- const durable = entry.durableAnalysis;
175
- if (durable === null) return terms;
176
- const {
177
- language,
178
- tokenizer,
179
- stopWords,
180
- stopWordList,
181
- analysisRevision: _analysisRevision,
182
- arrayPositionGap: _arrayPositionGap,
183
- analysisGeneration: _analysisGeneration,
184
- ...rest
185
- } = terms;
186
- return {
187
- ...rest,
188
- language: durable.language,
189
- ...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
190
- ...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
191
- ...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
192
- ...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
193
- ...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
194
- ...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
195
- pendingAnalysis: {
196
- generation: entry.analysisGeneration,
197
- language,
198
- ...tokenizer !== void 0 ? { tokenizer } : {},
199
- ...stopWords !== void 0 ? { stopWords } : {},
200
- ...stopWordList !== void 0 ? { stopWordList } : {}
201
- }
202
- };
203
- }
204
- function indexMetadataOf(indexName, entry, partitionCount) {
205
- return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
206
- }
207
-
208
- // src/serialization/stored-vector-values.ts
209
- function dropStoredVectorValues(documents, schema) {
210
- const vectorFieldPaths = [];
211
- for (const [fieldPath, fieldType] of Object.entries(schema)) {
212
- if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
213
- vectorFieldPaths.push(fieldPath);
214
- }
215
- }
216
- if (vectorFieldPaths.length === 0) {
217
- return;
218
- }
219
- for (const document of Object.values(documents)) {
220
- for (const fieldPath of vectorFieldPaths) {
221
- removeFieldAtPath(document.fields, fieldPath);
222
- }
223
- }
224
- }
225
- function removeFieldAtPath(fields, fieldPath) {
226
- if (!fieldPath.includes(".")) {
227
- delete fields[fieldPath];
228
- return;
229
- }
230
- const segments = fieldPath.split(".");
231
- let current = fields;
232
- for (let i = 0; i < segments.length - 1; i += 1) {
233
- const next = current[segments[i]];
234
- if (next === null || typeof next !== "object") {
235
- return;
236
- }
237
- current = next;
238
- }
239
- delete current[segments[segments.length - 1]];
240
- }
241
-
242
- // src/serialization/payload-v1/partition.ts
243
- function sanitizeSurfaceForms(raw) {
244
- if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
245
- const result = /* @__PURE__ */ Object.create(null);
246
- let count = 0;
247
- for (const [surface, value] of Object.entries(raw)) {
248
- if (typeof value === "number") {
249
- result[surface] = value;
250
- count++;
251
- } else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
252
- result[surface] = [value[0], value[1]];
253
- count++;
254
- }
255
- }
256
- return count > 0 ? result : void 0;
257
- }
258
- function isWireRecord(value) {
259
- return value !== null && typeof value === "object" && !Array.isArray(value);
260
- }
261
- function corruptPatternEntry(field) {
262
- return new NarsilError(
263
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
264
- `The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
265
- { field }
266
- );
267
- }
268
- function patternIndexesFromWire(raw) {
269
- if (raw === void 0 || raw === null) return {};
270
- if (!isWireRecord(raw)) {
271
- throw new NarsilError(
272
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
273
- "The pattern entry of this partition must be a map of fields"
274
- );
275
- }
276
- const pattern = /* @__PURE__ */ Object.create(null);
277
- for (const [field, index] of Object.entries(raw)) {
278
- if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
279
- throw corruptPatternEntry(field);
280
- }
281
- const docIds = [];
282
- for (const docId of index.doc_ids) {
283
- if (typeof docId !== "string") throw corruptPatternEntry(field);
284
- docIds.push(docId);
285
- }
286
- pattern[field] = { docIds, runs: index.runs };
287
- }
288
- return { pattern };
289
- }
290
- function validateHnswMetric(value) {
291
- if (value === void 0 || value === null) return void 0;
292
- if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
293
- return value;
294
- }
295
- return void 0;
296
- }
297
- function wireToPartition(raw) {
298
- const documents = {};
299
- for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
300
- documents[docId] = {
301
- fields: doc.fields,
302
- fieldLengths: doc.field_lengths ?? {}
303
- };
304
- }
305
- requireValidFieldTypes(raw.schema ?? {});
306
- dropStoredVectorValues(documents, raw.schema ?? {});
307
- const invertedIndex = {};
308
- for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
309
- invertedIndex[token] = {
310
- docFrequency: list.doc_freq,
311
- postings: (list.postings ?? []).map((p) => ({
312
- docId: p.doc_id,
313
- termFrequency: p.term_freq,
314
- field: p.field,
315
- positions: p.positions ?? []
316
- }))
317
- };
318
- }
319
- const numeric = {};
320
- for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
321
- numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
322
- }
323
- const boolean = {};
324
- for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
325
- boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
326
- }
327
- const geopoint = {};
328
- for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
329
- geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
330
- }
331
- const vectorData = {};
332
- for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
333
- vectorData[field] = {
334
- dimension: data.dimension,
335
- vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
336
- hnswGraph: data.hnsw_graph ? {
337
- entryPoint: data.hnsw_graph.entry_point,
338
- maxLayer: data.hnsw_graph.max_layer,
339
- m: data.hnsw_graph.m,
340
- efConstruction: data.hnsw_graph.ef_construction,
341
- metric: validateHnswMetric(data.hnsw_graph.metric),
342
- nodes: data.hnsw_graph.nodes
343
- } : null,
344
- codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
345
- };
346
- }
347
- return {
348
- indexName: raw.index_name,
349
- partitionId: raw.partition_id,
350
- totalPartitions: raw.total_partitions,
351
- language: raw.language ?? "english",
352
- schema: raw.schema ?? {},
353
- docCount: raw.doc_count ?? 0,
354
- avgDocLength: raw.avg_doc_length ?? 0,
355
- documents,
356
- invertedIndex,
357
- fieldIndexes: {
358
- numeric,
359
- boolean,
360
- enum: raw.field_indexes?.enum ?? {},
361
- geopoint,
362
- ...patternIndexesFromWire(raw.field_indexes?.pattern)
363
- },
364
- surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
365
- vectorData,
366
- statistics: {
367
- totalDocuments: raw.statistics?.total_documents ?? 0,
368
- totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
369
- averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
370
- docFrequencies: raw.statistics?.doc_frequencies ?? {}
371
- }
372
- };
373
- }
374
- function deserializePayloadV1(data) {
375
- const raw = decode(data);
376
- return wireToPartition(raw);
377
- }
378
-
379
- // src/serialization/payload-v1/sync-metadata.ts
380
- function invalid(message, details) {
381
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The device sync state in the index metadata ${message}`, details);
382
- }
383
- function isPositiveSafeInteger(value) {
384
- return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
385
- }
386
- function syncMetadataToWire(meta) {
387
- const wire = {};
388
- if (meta.sync !== void 0) {
389
- wire.sync = {
390
- enabled: meta.sync.enabled,
391
- ...meta.sync.historyMaxAgeMs !== void 0 ? { history_max_age_ms: meta.sync.historyMaxAgeMs } : {},
392
- ...meta.sync.historyMaxBytes !== void 0 ? { history_max_bytes: meta.sync.historyMaxBytes } : {}
393
- };
394
- }
395
- if (meta.syncActive !== void 0) wire.sync_active = meta.syncActive;
396
- if (meta.syncEpoch !== void 0) wire.sync_epoch = meta.syncEpoch;
397
- return wire;
398
- }
399
- function storedSyncSetting(raw) {
400
- if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
401
- const { enabled, history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
402
- if (typeof enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
403
- if (maxAge !== void 0 && !isPositiveSafeInteger(maxAge)) {
404
- invalid("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
405
- }
406
- if (maxBytes !== void 0 && !isPositiveSafeInteger(maxBytes)) {
407
- invalid("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
408
- }
409
- return {
410
- enabled,
411
- ...maxAge !== void 0 ? { historyMaxAgeMs: maxAge } : {},
412
- ...maxBytes !== void 0 ? { historyMaxBytes: maxBytes } : {}
413
- };
414
- }
415
- function syncMetadataFromWire(raw, meta) {
416
- if (raw.sync !== void 0 && raw.sync !== null) meta.sync = storedSyncSetting(raw.sync);
417
- if (raw.sync_active !== void 0 && raw.sync_active !== null) {
418
- if (typeof raw.sync_active !== "boolean") invalid("holds a sync_active that is not a boolean", {});
419
- meta.syncActive = raw.sync_active;
420
- }
421
- if (raw.sync_epoch !== void 0 && raw.sync_epoch !== null) {
422
- if (!isSyncEpoch(raw.sync_epoch)) invalid("holds an epoch that is not 32 lowercase hexadecimal digits", {});
423
- meta.syncEpoch = raw.sync_epoch;
424
- }
425
- }
426
-
427
- // src/serialization/payload-v1/metadata.ts
428
- function metadataToWire(meta) {
429
- const wire = {
430
- index_name: meta.indexName,
431
- schema: meta.schema,
432
- language: meta.language,
433
- partition_count: meta.partitionCount,
434
- bm25_params: meta.bm25Params,
435
- created_at: meta.createdAt,
436
- engine_version: meta.engineVersion
437
- };
438
- if (meta.vectorFields) {
439
- wire.vector_fields = meta.vectorFields;
440
- }
441
- if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
442
- wire.document_count = meta.documentCount;
443
- }
444
- if (meta.embedding) {
445
- wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
446
- }
447
- if (meta.surfaceForms !== void 0) {
448
- wire.surface_forms_enabled = meta.surfaceForms;
449
- }
450
- if (meta.analysisRevision !== void 0) {
451
- wire.analysis_revision = meta.analysisRevision;
452
- }
453
- if (meta.arrayPositionGap !== void 0) {
454
- wire.array_position_gap = meta.arrayPositionGap;
455
- }
456
- if (meta.tokenizer !== void 0) {
457
- wire.tokenizer = meta.tokenizer;
458
- }
459
- if (meta.stopWords !== void 0) {
460
- wire.stop_words = meta.stopWords;
461
- }
462
- if (meta.stopWordList !== void 0) {
463
- wire.stop_word_list = meta.stopWordList;
464
- }
465
- if (meta.analysisGeneration !== void 0) {
466
- wire.analysis_generation = meta.analysisGeneration;
467
- }
468
- if (meta.pendingAnalysis !== void 0) {
469
- const pending = meta.pendingAnalysis;
470
- wire.pending_analysis = {
471
- generation: pending.generation,
472
- language: pending.language,
473
- ...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
474
- ...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
475
- ...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
476
- };
477
- }
478
- if (meta.partitionLimits !== void 0) {
479
- wire.partition_limits = {
480
- ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
481
- ...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
482
- ...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
483
- };
484
- }
485
- if (meta.defaultScoring !== void 0) {
486
- wire.default_scoring = meta.defaultScoring;
487
- }
488
- if (meta.trackPositions !== void 0) {
489
- wire.track_positions = meta.trackPositions;
490
- }
491
- if (meta.textStorage !== void 0) {
492
- wire.text_storage = meta.textStorage;
493
- }
494
- if (meta.strict !== void 0) {
495
- wire.strict = meta.strict;
496
- }
497
- if (meta.required !== void 0) {
498
- wire.required = meta.required;
499
- }
500
- if (meta.vectorPromotion !== void 0) {
501
- const promotion = meta.vectorPromotion;
502
- wire.vector_promotion = {
503
- ...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
504
- ...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
505
- ...promotion.hnswConfig !== void 0 ? {
506
- hnsw_config: {
507
- ...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
508
- ...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
509
- ...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
510
- }
511
- } : {},
512
- ...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
513
- ...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
514
- };
515
- }
516
- if (meta.patternValueLimit !== void 0) {
517
- wire.pattern_value_limit = meta.patternValueLimit;
518
- }
519
- if (meta.indexUuid !== void 0) {
520
- wire.index_uuid = meta.indexUuid;
521
- }
522
- if (meta.heldPartitions !== void 0) {
523
- wire.held_partitions = meta.heldPartitions;
524
- }
525
- if (Array.isArray(meta.synonyms) && meta.synonyms.length > 0) {
526
- wire.synonyms = meta.synonyms;
527
- }
528
- return {
529
- ...wire,
530
- ...syncMetadataToWire(meta),
531
- ...meta.querySuggestions !== void 0 ? storedStateToWire(meta.querySuggestions) : {}
532
- };
533
- }
534
- function wireToMetadata(raw) {
535
- const meta = {
536
- indexName: raw.index_name,
537
- schema: raw.schema ?? {},
538
- language: raw.language ?? "english",
539
- partitionCount: raw.partition_count ?? 1,
540
- bm25Params: raw.bm25_params ?? { k1: DEFAULT_BM25_K1, b: DEFAULT_BM25_B },
541
- createdAt: raw.created_at ?? 0,
542
- engineVersion: raw.engine_version ?? "0.0.0"
543
- };
544
- if (raw.vector_fields) {
545
- for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
546
- if (!isQuantizationMode(field.quantization)) {
547
- throw new NarsilError(
548
- ErrorCodes.CONFIG_INVALID,
549
- `Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
550
- { fieldPath, quantization: field.quantization }
551
- );
552
- }
553
- }
554
- meta.vectorFields = raw.vector_fields;
555
- }
556
- if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
557
- meta.documentCount = raw.document_count;
558
- }
559
- if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
560
- meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
561
- }
562
- if (typeof raw.surface_forms_enabled === "boolean") {
563
- meta.surfaceForms = raw.surface_forms_enabled;
564
- }
565
- if (typeof raw.analysis_revision === "string") {
566
- meta.analysisRevision = raw.analysis_revision;
567
- }
568
- if (typeof raw.array_position_gap === "number") {
569
- meta.arrayPositionGap = raw.array_position_gap;
570
- }
571
- if (typeof raw.tokenizer === "string") {
572
- meta.tokenizer = raw.tokenizer;
573
- }
574
- if (typeof raw.stop_words === "string") {
575
- meta.stopWords = raw.stop_words;
576
- }
577
- if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
578
- meta.stopWordList = raw.stop_word_list;
579
- }
580
- if (isPositiveInteger(raw.analysis_generation)) {
581
- meta.analysisGeneration = raw.analysis_generation;
582
- }
583
- const pending = raw.pending_analysis;
584
- if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
585
- meta.pendingAnalysis = {
586
- generation: pending.generation,
587
- language: pending.language,
588
- ...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
589
- ...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
590
- ...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
591
- };
592
- }
593
- if (raw.partition_limits && typeof raw.partition_limits === "object") {
594
- const limits = {
595
- ...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
596
- ...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
597
- ...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
598
- };
599
- if (Object.keys(limits).length > 0) {
600
- meta.partitionLimits = limits;
601
- }
602
- }
603
- if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
604
- meta.defaultScoring = raw.default_scoring;
605
- }
606
- if (typeof raw.track_positions === "boolean") {
607
- meta.trackPositions = raw.track_positions;
608
- }
609
- if (raw.text_storage !== void 0) {
610
- if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
611
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
612
- textStorage: raw.text_storage
613
- });
614
- }
615
- meta.textStorage = raw.text_storage;
616
- }
617
- if (typeof raw.strict === "boolean") {
618
- meta.strict = raw.strict;
619
- }
620
- if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
621
- meta.required = raw.required;
622
- }
623
- if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
624
- const promotion = raw.vector_promotion;
625
- const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
626
- const metric = hnsw?.metric;
627
- const restored = {};
628
- const hnswConfig = {};
629
- if (isPositiveInteger(hnsw?.m)) {
630
- hnswConfig.m = hnsw?.m;
631
- }
632
- if (isPositiveInteger(hnsw?.ef_construction)) {
633
- hnswConfig.efConstruction = hnsw?.ef_construction;
634
- }
635
- if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
636
- hnswConfig.metric = metric;
637
- }
638
- if (isPositiveInteger(promotion.threshold)) {
639
- restored.threshold = promotion.threshold;
640
- }
641
- if (isSelectivityRatio(promotion.filter_threshold)) {
642
- restored.filterThreshold = promotion.filter_threshold;
643
- }
644
- if (Object.keys(hnswConfig).length > 0) {
645
- restored.hnswConfig = hnswConfig;
646
- }
647
- if (promotion.quantization !== void 0) {
648
- if (!isQuantizationMode(promotion.quantization)) {
649
- throw new NarsilError(
650
- ErrorCodes.CONFIG_INVALID,
651
- `Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
652
- { quantization: promotion.quantization }
653
- );
654
- }
655
- restored.quantization = promotion.quantization;
656
- }
657
- if (promotion.storage !== void 0) {
658
- if (!isStorageMode(promotion.storage)) {
659
- throw new NarsilError(
660
- ErrorCodes.CONFIG_INVALID,
661
- `Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
662
- { storage: promotion.storage }
663
- );
664
- }
665
- restored.storage = promotion.storage;
666
- }
667
- if (Object.keys(restored).length > 0) {
668
- meta.vectorPromotion = restored;
669
- }
670
- }
671
- if (raw.pattern_value_limit !== void 0) {
672
- if (!isPositiveInteger(raw.pattern_value_limit)) {
673
- throw new NarsilError(
674
- ErrorCodes.CONFIG_INVALID,
675
- `The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
676
- { patternValueLimit: String(raw.pattern_value_limit) }
677
- );
678
- }
679
- meta.patternValueLimit = raw.pattern_value_limit;
680
- }
681
- if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
682
- meta.indexUuid = raw.index_uuid;
683
- }
684
- if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
685
- meta.heldPartitions = raw.held_partitions;
686
- }
687
- if (raw.synonyms !== void 0 && raw.synonyms !== null) meta.synonyms = raw.synonyms;
688
- syncMetadataFromWire(raw, meta);
689
- const querySuggestions = storedStateFromWire(raw);
690
- if (querySuggestions !== null) meta.querySuggestions = querySuggestions;
691
- return meta;
692
- }
693
- function isNonNegativeInteger(value) {
694
- return typeof value === "number" && Number.isInteger(value) && value >= 0;
695
- }
696
- function isStringList(value) {
697
- return Array.isArray(value) && value.every((word) => typeof word === "string");
698
- }
699
- function isPositiveInteger(value) {
700
- return typeof value === "number" && Number.isInteger(value) && value >= 1;
701
- }
702
- function isSelectivityRatio(value) {
703
- return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
704
- }
705
- function isWatermarkFraction(value) {
706
- return typeof value === "number" && value > 0 && value <= 1;
707
- }
708
- function serializeMetadata(meta) {
709
- const wire = metadataToWire(meta);
710
- return encode(wire);
711
- }
712
- function deserializeMetadata(data) {
713
- const raw = decode(data);
714
- return wireToMetadata(raw);
715
- }
716
-
717
- // src/serialization/envelope.ts
718
- var CURRENT_ENVELOPE_VERSION = 1;
719
- var SNAPSHOT_ENVELOPE_VERSION = 2;
720
- var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
721
- function buildFlags(options) {
722
- const useCompression = options.compression === "gzip";
723
- return {
724
- compressionEnabled: useCompression,
725
- compressionAlgorithm: useCompression ? "gzip" : "none",
726
- checksumPresent: options.checksum ?? false,
727
- encryptionEnabled: false
728
- };
729
- }
730
- async function packEnvelope(payloadBytes, options) {
731
- const flags = buildFlags(options);
732
- let finalPayload = payloadBytes;
733
- if (flags.compressionEnabled) {
734
- finalPayload = await gzip(payloadBytes);
735
- }
736
- const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
737
- const [major, minor, patch] = engineVersionParts();
738
- const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
739
- const header = {
740
- magic: "NRSL",
741
- envelopeFormatVersion,
742
- engineVersionMajor: major,
743
- engineVersionMinor: minor,
744
- engineVersionPatch: patch,
745
- payloadLength: finalPayload.length,
746
- flags,
747
- checksum,
748
- reserved: new Uint8Array(14)
749
- };
750
- const headerBytes = writeHeader(header);
751
- const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
752
- envelope.set(headerBytes, 0);
753
- envelope.set(finalPayload, HEADER_SIZE);
754
- return envelope;
755
- }
756
- async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
757
- const header = readHeader(data);
758
- if (header.envelopeFormatVersion > maxAcceptedVersion) {
759
- throw new NarsilError(
760
- ErrorCodes.ENVELOPE_VERSION_MISMATCH,
761
- `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
762
- );
763
- }
764
- if (header.flags.encryptionEnabled) {
765
- throw new NarsilError(
766
- ErrorCodes.ENVELOPE_VERSION_MISMATCH,
767
- "Encrypted envelopes are not supported in this version of Narsil"
768
- );
769
- }
770
- const compAlgo = header.flags.compressionAlgorithm;
771
- if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
772
- throw new NarsilError(
773
- ErrorCodes.ENVELOPE_VERSION_MISMATCH,
774
- `Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
775
- );
776
- }
777
- const availablePayload = data.length - HEADER_SIZE;
778
- if (availablePayload < header.payloadLength) {
779
- throw new NarsilError(
780
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
781
- `Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
782
- );
783
- }
784
- let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
785
- if (header.flags.checksumPresent) {
786
- const computed = crc32(payloadBytes);
787
- if (computed !== header.checksum) {
788
- throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
789
- expected: header.checksum,
790
- computed
791
- });
792
- }
793
- }
794
- if (header.flags.compressionEnabled) {
795
- payloadBytes = await gunzip(payloadBytes);
796
- }
797
- return { header, payloadBytes };
798
- }
799
- async function unpackEnvelopeBytes(data) {
800
- return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
801
- }
802
- async function unpackIndexSnapshotEnvelope(data) {
803
- const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
804
- if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
805
- throw new NarsilError(
806
- ErrorCodes.ENVELOPE_VERSION_MISMATCH,
807
- `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
808
- );
809
- }
810
- return payloadBytes;
811
- }
812
- async function writeMetadataEnvelope(meta, options = {}) {
813
- const payloadBytes = serializeMetadata(meta);
814
- return packEnvelope(payloadBytes, options);
815
- }
816
- async function readMetadataEnvelope(data) {
817
- const { header, payloadBytes } = await unpackEnvelope(data);
818
- const metadata = deserializeMetadata(payloadBytes);
819
- return { header, metadata };
820
- }
821
-
822
- // src/persistence/durability/segment/captured-partition.ts
823
- function keepMaskOf(segment) {
824
- const keep = new Uint8Array(segment.arrays.documentCount);
825
- for (let ordinal = 0; ordinal < keep.length; ordinal++) keep[ordinal] = segment.isTombstoned(ordinal) ? 0 : 1;
826
- return keep;
827
- }
828
- function selectionOfMask(segment, keep) {
829
- return { segment, keep: (ordinal) => keep[ordinal] === 1 };
830
- }
831
- function frozenLiveTail(live, shared) {
832
- if (live.count() === 0) return null;
833
- const payload = live.encodeSegment();
834
- const documents = survivorDocumentsOf(partitionStateOf(live), payload);
835
- if (shared) {
836
- const snapshot = freezeSegmentShared(payload, documents);
837
- if (snapshot !== null) return createSharedFrozenSegment(snapshot);
838
- }
839
- return createFrozenSegment(payload, documents);
840
- }
841
- function frozenSegmentsOf(partition, shared) {
842
- const segments = [];
843
- const live = isCompositePartition(partition) ? partition.live : partition;
844
- if (isCompositePartition(partition)) {
845
- const ids = partition.frozenSegmentSizes().map((size) => size.segmentId);
846
- segments.push(...partition.frozenSegmentsById(ids));
847
- }
848
- const tail = frozenLiveTail(live, shared);
849
- if (tail !== null) segments.push(tail);
850
- return segments;
851
- }
852
- function partitionContentsOf(manager, partitionId, captureRecords) {
853
- const partition = manager.getPartition(partitionId);
854
- const segments = frozenSegmentsOf(partition, false);
855
- const selections = segments.map((segment) => selectionOfMask(segment, keepMaskOf(segment)));
856
- const contents = { partitionId, docCount: partition.count(), selections };
857
- if (captureRecords !== void 0) contents.writeRecords = captureRecords(segments);
858
- return contents;
859
- }
860
- function captureWholePartition(manager, partitionId, captureRecords) {
861
- if (typeof SharedArrayBuffer !== "function") return null;
862
- const partition = manager.getPartition(partitionId);
863
- if (!isCompositePartition(partition) || partition.frozenSegmentCount() === 0) return null;
864
- const frozen = frozenSegmentsOf(partition, true);
865
- const records = captureRecords?.(frozen);
866
- const segments = [];
867
- for (const segment of frozen) {
868
- if (segment.sharedSnapshot === null) return null;
869
- const captured = { snapshot: segment.sharedSnapshot, keep: keepMaskOf(segment) };
870
- const held = records?.get(segment);
871
- if (held !== void 0) captured.writeRecords = held;
872
- segments.push(captured);
873
- }
874
- return { partitionId, docCount: partition.count(), segments };
875
- }
876
- function contentsOfCaptured(captured) {
877
- const writeRecords = /* @__PURE__ */ new Map();
878
- const selections = captured.segments.map(({ snapshot, keep, writeRecords: held }) => {
879
- const segment = createSharedFrozenSegment(snapshot);
880
- if (held !== void 0) writeRecords.set(segment, held);
881
- return selectionOfMask(segment, keep);
882
- });
883
- const contents = { partitionId: captured.partitionId, docCount: captured.docCount, selections };
884
- if (writeRecords.size > 0) contents.writeRecords = writeRecords;
885
- return contents;
886
- }
887
-
888
- // src/serialization/crc32-combine.ts
889
- var IEEE_POLYNOMIAL = 3988292384;
890
- var CRC_BITS = 32;
891
- var zeroByteOperators = [];
892
- function timesVector(operator, vector) {
893
- let sum = 0;
894
- let rest = vector >>> 0;
895
- for (let row = 0; rest !== 0; row++, rest >>>= 1) {
896
- if (rest & 1) sum ^= operator[row];
897
- }
898
- return sum >>> 0;
899
- }
900
- function squared(operator) {
901
- const result = new Uint32Array(CRC_BITS);
902
- for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
903
- return result;
904
- }
905
- function firstOperator() {
906
- let oneZeroBit = new Uint32Array(CRC_BITS);
907
- oneZeroBit[0] = IEEE_POLYNOMIAL;
908
- for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
909
- for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
910
- return oneZeroBit;
911
- }
912
- function operatorForZeroBytesPower(power) {
913
- if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
914
- while (zeroByteOperators.length <= power) {
915
- zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
916
- }
917
- return zeroByteOperators[power];
918
- }
919
- function crc32Combine(first, second, secondByteLength) {
920
- let crc = first >>> 0;
921
- let rest = secondByteLength;
922
- for (let power = 0; rest > 0; power++) {
923
- if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
924
- rest = Math.floor(rest / 2);
925
- }
926
- return (crc ^ second) >>> 0;
927
- }
928
- var BIN8 = 196;
929
- var BIN16 = 197;
930
- var BIN32 = 198;
931
- var ARRAY16 = 220;
932
- var ARRAY32 = 221;
933
- var FIXMAP = 128;
934
- var FIXARRAY = 144;
935
- var FIXED_LIST_MAX = 15;
936
- var FIXSTR = 160;
937
- var FIXSTR_MAX_LENGTH = 31;
938
- var ASCII_MAX = 127;
939
- var UTF8_MAX_BYTES_PER_UNIT = 3;
940
- var STRING_HEADER_MAX_BYTES = 5;
941
- var BIN8_MAX_BYTES = 255;
942
- var UINT16_MAX = 65535;
943
- function binHeader(byteLength) {
944
- if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
945
- if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
946
- const header = new Uint8Array(5);
947
- header[0] = BIN32;
948
- new DataView(header.buffer).setUint32(1, byteLength, false);
949
- return header;
950
- }
951
- function arrayHeader(count) {
952
- if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
953
- if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
954
- const header = new Uint8Array(5);
955
- header[0] = ARRAY32;
956
- new DataView(header.buffer).setUint32(1, count, false);
957
- return header;
958
- }
959
- function createPayloadBuilder() {
960
- const chunks = [];
961
- let offset = 0;
962
- function push(chunk) {
963
- chunks.push(chunk);
964
- offset += chunk.byteLength;
965
- }
966
- return {
967
- chunks,
968
- get offset() {
969
- return offset;
970
- },
971
- push,
972
- entry(key, value) {
973
- push(encode(key));
974
- push(encode(value));
975
- },
976
- rawEntry(key, encoded) {
977
- push(encode(key));
978
- const at = offset;
979
- push(encoded);
980
- return at;
981
- },
982
- bytesEntry(key, bytes) {
983
- push(encode(key));
984
- push(binHeader(bytes.byteLength));
985
- const at = offset;
986
- push(bytes);
987
- return at;
988
- }
989
- };
990
- }
991
- function pageChecksums(bytes, pageBytes) {
992
- const pages = Math.ceil(bytes.byteLength / pageBytes);
993
- const checksums = new Uint8Array(pages * 4);
994
- const view = new DataView(checksums.buffer);
995
- for (let page = 0; page < pages; page++) {
996
- const from = page * pageBytes;
997
- view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
998
- }
999
- return checksums;
1000
- }
1001
- function sectionOf(name, offset, bytes, pageBytes) {
1002
- return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
1003
- }
1004
- function uint32Bytes(values) {
1005
- const bytes = new Uint8Array(values.length * 4);
1006
- const view = new DataView(bytes.buffer);
1007
- for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
1008
- return bytes;
1009
- }
1010
- function writtenAsShortAscii(docId, list, at) {
1011
- if (docId.length > FIXSTR_MAX_LENGTH) return false;
1012
- for (let unit = 0; unit < docId.length; unit++) {
1013
- const code = docId.charCodeAt(unit);
1014
- if (code > ASCII_MAX) return false;
1015
- list[at + 1 + unit] = code;
1016
- }
1017
- list[at] = FIXSTR | docId.length;
1018
- return true;
1019
- }
1020
- function encodedIdList(docIds) {
1021
- const header = arrayHeader(docIds.length);
1022
- let capacity = header.byteLength;
1023
- for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
1024
- const list = new Uint8Array(capacity);
1025
- list.set(header);
1026
- const offsets = new Uint32Array(docIds.length + 1);
1027
- let cursor = header.byteLength;
1028
- for (let i = 0; i < docIds.length; i++) {
1029
- offsets[i] = cursor;
1030
- if (writtenAsShortAscii(docIds[i], list, cursor)) {
1031
- cursor += docIds[i].length + 1;
1032
- continue;
1033
- }
1034
- const encoded = encode(docIds[i]);
1035
- list.set(encoded, cursor);
1036
- cursor += encoded.byteLength;
1037
- }
1038
- offsets[docIds.length] = cursor;
1039
- return { list: list.subarray(0, cursor), offsets };
1040
- }
1041
- function positionsInIdOrder(docIds) {
1042
- const positions = Array.from(docIds.keys());
1043
- positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
1044
- return Uint32Array.from(positions);
1045
- }
1046
- function magnitudeBytes(magnitudes) {
1047
- const bytes = new Uint8Array(magnitudes.length * 8);
1048
- const view = new DataView(bytes.buffer);
1049
- for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
1050
- return bytes;
1051
- }
1052
- function checksumAcrossPages(checksums, byteLength, pageBytes) {
1053
- const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
1054
- let whole = 0;
1055
- for (let page = 0; page * 4 < checksums.byteLength; page++) {
1056
- const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
1057
- whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
1058
- }
1059
- return whole;
1060
- }
1061
- function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
1062
- let headLength = 0;
1063
- let head = new Uint8Array(0);
1064
- let vectorsSection = {
1065
- name: "vectors",
1066
- offset: 0,
1067
- length: vectors.byteLength,
1068
- pageBytes: SEGMENT_PAGE_BYTES,
1069
- checksums
1070
- };
1071
- for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
1072
- vectorsSection = {
1073
- ...vectorsSection,
1074
- offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
1075
- };
1076
- head = encode({ sections: [...sections, vectorsSection] });
1077
- if (head.byteLength === headLength) break;
1078
- headLength = head.byteLength;
1079
- }
1080
- return { head, vectorsSection };
1081
- }
1082
- function pagedVectorFileChunks(payload, heldMagnitudes) {
1083
- const builder = createPayloadBuilder();
1084
- const { codes } = payload;
1085
- builder.push(Uint8Array.of(FIXMAP | 10));
1086
- builder.entry("v", payload.v);
1087
- builder.entry("fieldName", payload.fieldName);
1088
- builder.entry("dimension", payload.dimension);
1089
- const { list, offsets } = encodedIdList(payload.docIds);
1090
- const docIdsAt = builder.rawEntry("docIds", list);
1091
- const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
1092
- builder.push(encode("codes"));
1093
- if (codes === null) {
1094
- builder.push(encode(null));
1095
- } else {
1096
- builder.push(Uint8Array.of(FIXMAP | 3));
1097
- builder.entry("bits", codes.bits);
1098
- builder.entry("centroid", codes.centroid);
1099
- const recordsAt = builder.bytesEntry("records", codes.records);
1100
- sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
1101
- }
1102
- const magnitudes = magnitudeBytes(heldMagnitudes);
1103
- sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
1104
- const idOffsets = uint32Bytes(offsets);
1105
- sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
1106
- const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
1107
- sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
1108
- const headKey = encode("head");
1109
- const vectorsKey = encode("vectors");
1110
- const vectorsHeader = binHeader(payload.vectors.byteLength);
1111
- const beforeHead = builder.offset + headKey.byteLength;
1112
- const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
1113
- const { head } = encodedHeadAndVectorsPlace(
1114
- sections,
1115
- (headLength) => beforeHead + binHeader(headLength).byteLength,
1116
- payload.vectors,
1117
- vectorChecksums,
1118
- vectorsKey.byteLength + vectorsHeader.byteLength
1119
- );
1120
- const headOffset = builder.bytesEntry("head", head);
1121
- builder.push(vectorsKey);
1122
- builder.push(vectorsHeader);
1123
- const beforeVectors = [...builder.chunks];
1124
- builder.push(payload.vectors);
1125
- return {
1126
- chunks: builder.chunks,
1127
- headOffset,
1128
- headLength: head.byteLength,
1129
- headChecksum: crc32(head),
1130
- beforeVectors,
1131
- vectors: {
1132
- byteLength: payload.vectors.byteLength,
1133
- checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
1134
- }
1135
- };
1136
- }
1137
- function nodeOffsetsOf(graph) {
1138
- const offsets = new Uint32Array(graph.levels.length + 1);
1139
- const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
1140
- let cursor = 0;
1141
- for (let at = 0; at < graph.levels.length; at++) {
1142
- offsets[at] = cursor;
1143
- for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
1144
- }
1145
- offsets[graph.levels.length] = cursor;
1146
- return uint32Bytes(offsets);
1147
- }
1148
- function pagedGraphPartChunks(part) {
1149
- const builder = createPayloadBuilder();
1150
- builder.push(Uint8Array.of(FIXMAP | 4));
1151
- builder.entry("v", part.v);
1152
- builder.entry("fieldName", part.fieldName);
1153
- builder.push(encode("graphs"));
1154
- builder.push(arrayHeader(part.graphs.length));
1155
- const layouts = [];
1156
- for (const graph of part.graphs) {
1157
- const { levels, neighbours, ...header } = graph;
1158
- const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
1159
- builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
1160
- for (const [key, value] of headerEntries) builder.entry(key, value);
1161
- const levelsAt = builder.bytesEntry("levels", levels);
1162
- const neighboursAt = builder.bytesEntry("neighbours", neighbours);
1163
- layouts.push({
1164
- ...header,
1165
- levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
1166
- nodeOffsets: nodeOffsetsOf(graph),
1167
- neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
1168
- });
1169
- }
1170
- const head = encode({ graphs: layouts });
1171
- const headOffset = builder.bytesEntry("head", head);
1172
- return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
1173
- }
1174
- var VECTOR_FILE_ENVELOPE_VERSION = 2;
1175
- var GRAPH_PART_ENVELOPE_VERSION = 2;
1176
- function liveVectorsOf(ref) {
1177
- let live = 0;
1178
- for (const file of ref.files) live += file.count - countDead(file.dead, file.count);
1179
- return live;
1180
- }
1181
- async function openListedFile(store, ref, what, allowsCompression) {
1182
- const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
1183
- const { header, payload } = await openChecksummedEnvelope(bytes, {
1184
- envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
1185
- what: `${what} "${ref.key}"`,
1186
- allowsCompression
1187
- });
1188
- requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
1189
- return { payload, payloadLength: header.payloadLength };
1190
- }
1191
- async function readGraphParts(store, ref) {
1192
- if (ref.graphParts.length === 0) return null;
1193
- const parts = [];
1194
- for (const part of ref.graphParts) {
1195
- const { payload } = await openListedFile(store, part, "graph part", true);
1196
- const decoded = decodeVectorGraphPartPayload(decode(payload));
1197
- if (decoded.fieldName !== ref.fieldPath) {
1198
- throw new NarsilError(
1199
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1200
- `The graph part "${part.key}" names field "${decoded.fieldName}" where the manifest lists "${ref.fieldPath}"`,
1201
- { key: part.key }
1202
- );
1203
- }
1204
- parts.push(decoded);
1205
- }
1206
- return assembleGraphParts(parts);
1207
- }
1208
- async function readVectorFile(store, file, fieldPath) {
1209
- const { payload: bytes, payloadLength } = await openListedFile(store, file, "vector file", false);
1210
- const payload = decodeVectorFilePayload(decode(bytes));
1211
- if (payload.docIds.length !== file.count) {
1212
- throw new NarsilError(
1213
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1214
- `The vector file "${file.key}" holds ${payload.docIds.length} vectors where the manifest counts ${file.count}`,
1215
- { key: file.key, held: payload.docIds.length, count: file.count }
1216
- );
1217
- }
1218
- if (payload.fieldName !== fieldPath) {
1219
- throw new NarsilError(
1220
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1221
- `The vector file "${file.key}" names field "${payload.fieldName}" where the manifest lists "${fieldPath}"`,
1222
- { key: file.key }
1223
- );
1224
- }
1225
- return { payload, payloadLength };
1226
- }
1227
- function holdsEveryHead(ref) {
1228
- if (ref.graphParts.length === 0 || ref.files.length === 0) return false;
1229
- return [...ref.files, ...ref.graphParts].every((file) => file.headOffset !== void 0);
1230
- }
1231
- async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
1232
- const { pathOf } = store;
1233
- if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
1234
- const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-N6KFNNZA.mjs');
1235
- await Promise.all([
1236
- ...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
1237
- ...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
1238
- ]);
1239
- const files = await Promise.all(
1240
- ref.files.map(async (file) => ({ ref: file, path: await pathOf.call(store, file.key) }))
1241
- );
1242
- const graphParts = await Promise.all(
1243
- ref.graphParts.map(async (part) => ({ ref: part, path: await pathOf.call(store, part.key) }))
1244
- );
1245
- const field = openColdVectorField({
1246
- fieldPath: ref.fieldPath,
1247
- dimension: vectorIndex.dimension,
1248
- metric: vectorIndex.metric,
1249
- codeBits: osqBitsOf(vectorIndex.quantization),
1250
- files,
1251
- graphParts,
1252
- resolvePartition
1253
- });
1254
- if (field === null) return false;
1255
- vectorIndex.openColdField(field, async (beginRestore, stillWanted, loadingWanted) => {
1256
- await waitBeforeColdLoad(loadingWanted);
1257
- if (!stillWanted()) return;
1258
- const graph = await readGraphParts(store, ref);
1259
- if (!stillWanted()) return;
1260
- const restore = beginRestore({ liveVectors: liveVectorsOf(ref), holdsGraph: true, graph });
1261
- await loadFilesByParts(files, restore, ref.fieldPath, vectorIndex.dimension, stillWanted);
1262
- });
1263
- return true;
1264
- }
1265
- async function loadVectorField(store, ref, vectorIndex, resolvePartition = null) {
1266
- if (resolvePartition !== null && await loadFieldOnDemand(store, ref, vectorIndex, resolvePartition)) return;
1267
- const restore = vectorIndex.restoreCheckpoint({
1268
- liveVectors: liveVectorsOf(ref),
1269
- holdsGraph: ref.graphParts.length > 0,
1270
- graph: await readGraphParts(store, ref)
1271
- });
1272
- for (const file of ref.files) {
1273
- const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
1274
- restore.addFile({
1275
- id: file.id,
1276
- key: file.key,
1277
- payload,
1278
- dead: file.dead,
1279
- location: store.pathOf === void 0 ? null : {
1280
- path: await store.pathOf(file.key),
1281
- vectorsOffset: HEADER_SIZE + payloadLength - payload.vectors.byteLength
1282
- }
1283
- });
1284
- }
1285
- restore.finish();
1286
- }
1287
- var EMPTY_BIN = new Uint8Array(0);
1288
- var EMPTY_BIN_BYTES = 2;
1289
- var BIN82 = 196;
1290
- var BIN162 = 197;
1291
- var BIN322 = 198;
1292
- var BIN8_MAX_BYTES2 = 255;
1293
- var BIN16_MAX_BYTES = 65535;
1294
- function binHeader2(byteLength) {
1295
- if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
1296
- if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
1297
- const header = new Uint8Array(5);
1298
- header[0] = BIN322;
1299
- new DataView(header.buffer).setUint32(1, byteLength, false);
1300
- return header;
1301
- }
1302
- function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
1303
- const head = encode(withEmptyBytesInTheirPlace);
1304
- const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
1305
- if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
1306
- return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
1307
- }
1308
- function vectorPartChunks(part) {
1309
- return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
1310
- }
1311
-
1312
- // src/persistence/durability/segment/vector-write.ts
1313
- function keptFileRef(input, prior, kept) {
1314
- const listed = prior?.files.find((file) => file.key === kept.key) ?? input.priorFileRefs?.get(kept.key);
1315
- if (listed === void 0) {
1316
- throw new NarsilError(
1317
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1318
- `The checkpoint keeps the vector file "${kept.key}", which the previous manifest does not list`,
1319
- { key: kept.key }
1320
- );
1321
- }
1322
- return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
1323
- }
1324
- function headFieldsOf(ref) {
1325
- return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
1326
- }
1327
- function headRefOf(paged) {
1328
- return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
1329
- }
1330
- async function writeGraphParts(input, fieldPath, plan, generation) {
1331
- const parts = plan.graphParts ?? [];
1332
- const refs = [];
1333
- for (let part = 0; part < parts.length; part++) {
1334
- const key = graphPartKey(input.indexName, fieldPath, generation, part);
1335
- const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
1336
- const packed = await packChecksummedEnvelope(
1337
- paged === null ? [encode(parts[part])] : paged.chunks,
1338
- GRAPH_PART_ENVELOPE_VERSION,
1339
- `graph part "${key}"`
1340
- );
1341
- await input.store.write(key, packed.chunks);
1342
- refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
1343
- }
1344
- return refs;
1345
- }
1346
- async function writeField(input, fieldPath, plan, outcome) {
1347
- const prior = input.priorVectors.find((ref) => ref.fieldPath === fieldPath);
1348
- if (plan.unchanged && prior !== void 0) {
1349
- outcome.refs.push(prior);
1350
- return;
1351
- }
1352
- let nextFileId = prior?.nextFileId ?? 0;
1353
- const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
1354
- const written = [];
1355
- for (let index = 0; index < plan.newFiles; index++) {
1356
- const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
1357
- const id = nextFileId;
1358
- nextFileId += 1;
1359
- const key = vectorFileKey(input.indexName, fieldPath, id);
1360
- const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
1361
- const what = `vector file "${key}"`;
1362
- const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
1363
- paged.chunks,
1364
- VECTOR_FILE_ENVELOPE_VERSION,
1365
- what,
1366
- crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
1367
- );
1368
- await input.store.write(key, packed.chunks);
1369
- files.push({
1370
- id,
1371
- key,
1372
- count: payload.docIds.length,
1373
- dead,
1374
- bytes: packed.byteLength,
1375
- checksum: packed.checksum,
1376
- ...headRefOf(paged)
1377
- });
1378
- written.push({ id, key, ordinals });
1379
- outcome.layouts.push({
1380
- fieldPath,
1381
- key,
1382
- docIds: payload.docIds,
1383
- dead,
1384
- vectorsOffset: packed.byteLength - payload.vectors.byteLength
1385
- });
1386
- }
1387
- const priorGeneration = prior?.graphGeneration ?? 0;
1388
- const holdsGraph = plan.graphParts !== null && plan.graphParts.length > 0;
1389
- const graphGeneration = holdsGraph ? priorGeneration + 1 : priorGeneration;
1390
- const graphParts = holdsGraph ? await writeGraphParts(input, fieldPath, plan, graphGeneration) : [];
1391
- outcome.refs.push({ fieldPath, nextFileId, files, graphGeneration, graphParts });
1392
- outcome.written.push({ fieldPath, plan, files: written });
1393
- }
1394
- async function writeLiveVectors(input) {
1395
- const outcome = { refs: [], layouts: [], written: [] };
1396
- for (const [fieldPath, plan] of input.plans) await writeField(input, fieldPath, plan, outcome);
1397
- return outcome;
1398
- }
1399
-
1400
- // src/persistence/durability/seq-owner.ts
1401
- var SINGLE_NODE_PRIMARY_TERM = 1;
1402
- function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
1403
- let current = startSeqNo;
1404
- let term = primaryTerm;
1405
- return {
1406
- next() {
1407
- current += 1;
1408
- return current;
1409
- },
1410
- advanceTo(seqNo, nextPrimaryTerm) {
1411
- current = seqNo;
1412
- term = nextPrimaryTerm;
1413
- },
1414
- get current() {
1415
- return current;
1416
- },
1417
- get primaryTerm() {
1418
- return term;
1419
- }
1420
- };
1421
- }
1422
- function fileRefOnTheWire(ref) {
1423
- return {
1424
- key: ref.key,
1425
- bytes: ref.bytes,
1426
- checksum: ref.checksum,
1427
- ...ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum }
1428
- };
1429
- }
1430
- function encodeSegmentManifest(manifest) {
1431
- const payload = encode({
1432
- version: SEGMENT_MANIFEST_VERSION,
1433
- schema: manifest.schema,
1434
- language: manifest.language,
1435
- ...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
1436
- checkpoint: manifest.checkpoint.map((c) => ({
1437
- partitionId: c.partitionId,
1438
- lastSeqNo: c.lastSeqNo,
1439
- primaryTerm: c.primaryTerm,
1440
- ...c.termHistory === void 0 ? {} : {
1441
- termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
1442
- }
1443
- })),
1444
- partitions: manifest.partitions.map((p) => ({
1445
- partitionId: p.partitionId,
1446
- nextSegmentId: p.nextSegmentId,
1447
- segments: p.segments.map((s) => ({
1448
- id: s.id,
1449
- key: s.key,
1450
- docCount: s.docCount,
1451
- tombstoneCount: s.tombstoneCount,
1452
- bytes: s.bytes,
1453
- checksum: s.checksum
1454
- }))
1455
- })),
1456
- vectors: manifest.vectors.map((v) => ({
1457
- fieldPath: v.fieldPath,
1458
- nextFileId: v.nextFileId,
1459
- files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
1460
- graphGeneration: v.graphGeneration,
1461
- graphParts: v.graphParts.map(fileRefOnTheWire)
1462
- }))
1463
- });
1464
- return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
1465
- }
1466
-
1467
- // src/persistence/durability/partition-checkpoint.ts
1468
- function checkpointLastSeqNo(checkpoint, partitionId) {
1469
- for (const c of checkpoint) {
1470
- if (c.partitionId === partitionId) {
1471
- return c.lastSeqNo;
1472
- }
1473
- }
1474
- return 0;
1475
- }
1476
- function checkpointPrimaryTerm(checkpoint, partitionId) {
1477
- for (const c of checkpoint) {
1478
- if (c.partitionId === partitionId) {
1479
- return c.primaryTerm;
1480
- }
1481
- }
1482
- return 0;
1483
- }
1484
-
1485
- // src/persistence/durability/commit-marker.ts
1486
- var COMMIT_MARKER_SLOT_SIZE = 36;
1487
- var COMMIT_MARKER_SLOT_COUNT = 2;
1488
- var CRC_OFFSET = 32;
1489
- function encodeMarkerSlot(state) {
1490
- const slot = new Uint8Array(COMMIT_MARKER_SLOT_SIZE);
1491
- const view = new DataView(slot.buffer);
1492
- view.setBigUint64(0, BigInt(state.writeSeq), false);
1493
- view.setBigUint64(8, BigInt(state.activeSegmentSeqNo), false);
1494
- view.setBigUint64(16, BigInt(state.durableByteLength), false);
1495
- view.setBigUint64(24, BigInt(state.highestDurableSeqNo), false);
1496
- view.setUint32(CRC_OFFSET, crc32(slot.subarray(0, CRC_OFFSET)), false);
1497
- return slot;
1498
- }
1499
- function decodeSlot(data, slotIndex) {
1500
- const base = slotIndex * COMMIT_MARKER_SLOT_SIZE;
1501
- if (base + COMMIT_MARKER_SLOT_SIZE > data.length) {
1502
- return null;
1503
- }
1504
- const slot = data.subarray(base, base + COMMIT_MARKER_SLOT_SIZE);
1505
- const view = new DataView(slot.buffer, slot.byteOffset, slot.byteLength);
1506
- const storedCrc = view.getUint32(CRC_OFFSET, false);
1507
- if (crc32(slot.subarray(0, CRC_OFFSET)) !== storedCrc) {
1508
- return null;
1509
- }
1510
- const writeSeq = view.getBigUint64(0, false);
1511
- const activeSegmentSeqNo = view.getBigUint64(8, false);
1512
- const durableByteLength = view.getBigUint64(16, false);
1513
- const highestDurableSeqNo = view.getBigUint64(24, false);
1514
- if (writeSeq > BigInt(Number.MAX_SAFE_INTEGER) || activeSegmentSeqNo > BigInt(Number.MAX_SAFE_INTEGER) || durableByteLength > BigInt(Number.MAX_SAFE_INTEGER) || highestDurableSeqNo > BigInt(Number.MAX_SAFE_INTEGER)) {
1515
- return null;
1516
- }
1517
- return {
1518
- writeSeq: Number(writeSeq),
1519
- activeSegmentSeqNo: Number(activeSegmentSeqNo),
1520
- durableByteLength: Number(durableByteLength),
1521
- highestDurableSeqNo: Number(highestDurableSeqNo)
1522
- };
1523
- }
1524
- function readCommitMarker(data) {
1525
- let best = null;
1526
- for (let slotIndex = 0; slotIndex < COMMIT_MARKER_SLOT_COUNT; slotIndex += 1) {
1527
- const state = decodeSlot(data, slotIndex);
1528
- if (state === null) {
1529
- continue;
1530
- }
1531
- if (best === null || state.writeSeq > best.state.writeSeq) {
1532
- best = { state, slotIndex };
1533
- }
1534
- }
1535
- return best;
1536
- }
1537
- function nextMarkerSlot(lastWrittenSlot) {
1538
- if (lastWrittenSlot === null) {
1539
- return 0;
1540
- }
1541
- return (lastWrittenSlot + 1) % COMMIT_MARKER_SLOT_COUNT;
1542
- }
1543
- function markerSlotOffset(slotIndex) {
1544
- return slotIndex * COMMIT_MARKER_SLOT_SIZE;
1545
- }
1546
- function isRecord(value) {
1547
- return value !== null && typeof value === "object" && !Array.isArray(value);
1548
- }
1549
- function optionalIndexUuid(value, payloadName) {
1550
- if (value === void 0 || value === null) {
1551
- return void 0;
1552
- }
1553
- if (typeof value !== "string" || value.length === 0) {
1554
- throw new Error(`Invalid ${payloadName}: "indexUuid" must be a non-empty string or absent`);
1555
- }
1556
- return value;
1557
- }
1558
- function optionalPreviousPrimaryTerm(value, payloadName) {
1559
- if (value === void 0 || value === null) {
1560
- return void 0;
1561
- }
1562
- if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
1563
- throw new Error(`Invalid ${payloadName}: "previousPrimaryTerm" must be a non-negative integer or absent`);
1564
- }
1565
- return value;
1566
- }
1567
- function validateEntryPayload(decoded) {
1568
- if (!isRecord(decoded) || !isRecord(decoded.entry)) {
1569
- throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
1570
- }
1571
- const entry = decoded.entry;
1572
- if (typeof entry.seqNo !== "number") {
1573
- throw new Error('Invalid EntryPayload: "entry.seqNo" must be a number');
1574
- }
1575
- if (typeof entry.primaryTerm !== "number") {
1576
- throw new Error('Invalid EntryPayload: "entry.primaryTerm" must be a number');
1577
- }
1578
- if (entry.operation !== "INDEX" && entry.operation !== "DELETE") {
1579
- throw new Error('Invalid EntryPayload: "entry.operation" must be "INDEX" or "DELETE"');
1580
- }
1581
- if (typeof entry.partitionId !== "number") {
1582
- throw new Error('Invalid EntryPayload: "entry.partitionId" must be a number');
1583
- }
1584
- if (typeof entry.indexName !== "string") {
1585
- throw new Error('Invalid EntryPayload: "entry.indexName" must be a string');
1586
- }
1587
- if (typeof entry.documentId !== "string") {
1588
- throw new Error('Invalid EntryPayload: "entry.documentId" must be a string');
1589
- }
1590
- if (entry.document !== null && !(entry.document instanceof Uint8Array)) {
1591
- throw new Error('Invalid EntryPayload: "entry.document" must be Uint8Array or null');
1592
- }
1593
- if (typeof entry.checksum !== "number") {
1594
- throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
1595
- }
1596
- optionalIndexUuid(decoded.indexUuid, "EntryPayload");
1597
- optionalPreviousPrimaryTerm(decoded.previousPrimaryTerm, "EntryPayload");
1598
- return decoded;
1599
- }
1600
- function computeEntryChecksum(entry) {
1601
- const payload = encode([
1602
- entry.seqNo,
1603
- entry.primaryTerm,
1604
- entry.operation,
1605
- entry.partitionId,
1606
- entry.indexName,
1607
- entry.documentId,
1608
- entry.document
1609
- ]);
1610
- return crc32(payload);
1611
- }
1612
- function buildEntry(fields) {
1613
- return { ...fields, checksum: computeEntryChecksum(fields) };
1614
- }
1615
-
1616
- // src/persistence/durability/wal-framing.ts
1617
- var WAL_MAGIC = new Uint8Array([78, 82, 83, 87]);
1618
- var WAL_FORMAT_VERSION = 1;
1619
- var SEGMENT_HEADER_SIZE = 8;
1620
- var RECORD_LENGTH_SIZE = 4;
1621
- var FRAME_CRC_SIZE = 4;
1622
- var MAX_WAL_RECORD_BYTES = 268435456;
1623
- function writeSegmentHeader() {
1624
- const header = new Uint8Array(SEGMENT_HEADER_SIZE);
1625
- header.set(WAL_MAGIC, 0);
1626
- header[4] = WAL_FORMAT_VERSION & 255;
1627
- return header;
1628
- }
1629
- function frameRecord(entry) {
1630
- const payload = encode(entry);
1631
- const frame = new Uint8Array(RECORD_LENGTH_SIZE + payload.length + FRAME_CRC_SIZE);
1632
- const view = new DataView(frame.buffer);
1633
- view.setUint32(0, payload.length, false);
1634
- frame.set(payload, RECORD_LENGTH_SIZE);
1635
- view.setUint32(RECORD_LENGTH_SIZE + payload.length, crc32(payload), false);
1636
- return frame;
1637
- }
1638
- function checkSegmentHeader(data) {
1639
- if (data.length < SEGMENT_HEADER_SIZE) {
1640
- return { ok: false, reason: "segment shorter than the 8-byte header", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
1641
- }
1642
- for (let i = 0; i < WAL_MAGIC.length; i += 1) {
1643
- if (data[i] !== WAL_MAGIC[i]) {
1644
- return { ok: false, reason: "missing NRSW magic", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
1645
- }
1646
- }
1647
- const formatVersion = data[4];
1648
- if (formatVersion > WAL_FORMAT_VERSION) {
1649
- return {
1650
- ok: false,
1651
- reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}, so a newer release wrote this segment`,
1652
- code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
1653
- };
1654
- }
1655
- return { ok: true };
1656
- }
1657
- function decodeWalEntry(payload) {
1658
- const decoded = decode(payload);
1659
- const { entry } = validateEntryPayload({ entry: decoded });
1660
- if (computeEntryChecksum(entry) !== entry.checksum) {
1661
- throw new Error("WAL record failed its logical entry checksum");
1662
- }
1663
- return entry;
1664
- }
1665
- function corrupt(message, details) {
1666
- throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
1667
- }
1668
- function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
1669
- const header = checkSegmentHeader(data);
1670
- if (!header.ok) {
1671
- throw new NarsilError(header.code, `WAL segment header invalid in durable region: ${header.reason}`, {
1672
- reason: header.reason
1673
- });
1674
- }
1675
- if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
1676
- corrupt("Commit marker durable byte length is outside the segment", {
1677
- durableByteLength,
1678
- segmentLength: data.length
1679
- });
1680
- }
1681
- const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
1682
- const entries = [];
1683
- let offset = SEGMENT_HEADER_SIZE;
1684
- while (offset < durableByteLength) {
1685
- if (offset + RECORD_LENGTH_SIZE > durableByteLength) {
1686
- corrupt("WAL record length field overruns the durable region", { offset, durableByteLength });
1687
- }
1688
- const recordLength = view.getUint32(offset, false);
1689
- if (recordLength === 0 || recordLength > maxRecordBytes) {
1690
- corrupt("WAL record length is zero or exceeds the maximum record size", { offset, recordLength });
1691
- }
1692
- const payloadStart = offset + RECORD_LENGTH_SIZE;
1693
- const crcStart = payloadStart + recordLength;
1694
- const frameEnd = crcStart + FRAME_CRC_SIZE;
1695
- if (frameEnd > durableByteLength) {
1696
- corrupt("WAL record overruns the durable region", { offset, frameEnd, durableByteLength });
1697
- }
1698
- const payload = data.subarray(payloadStart, crcStart);
1699
- const storedCrc = view.getUint32(crcStart, false);
1700
- if (crc32(payload) !== storedCrc) {
1701
- corrupt("WAL record failed its frame checksum inside the durable region", { offset });
1702
- }
1703
- let entry;
1704
- try {
1705
- entry = decodeWalEntry(payload);
1706
- } catch (err) {
1707
- corrupt("WAL record payload failed to decode inside the durable region", {
1708
- offset,
1709
- cause: err instanceof Error ? err.message : String(err)
1710
- });
1711
- }
1712
- if (entries.length > 0 && entry.seqNo <= entries[entries.length - 1].seqNo) {
1713
- corrupt("WAL sequence numbers are not strictly increasing", {
1714
- previousSeqNo: entries[entries.length - 1].seqNo,
1715
- seqNo: entry.seqNo
1716
- });
1717
- }
1718
- entries.push(entry);
1719
- offset = frameEnd;
1720
- }
1721
- return entries;
1722
- }
1723
- function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
1724
- const start = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
1725
- const entries = [];
1726
- if (start >= data.length) {
1727
- return { entries, cleanEnd: start };
1728
- }
1729
- const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
1730
- let offset = start;
1731
- let cleanEnd = start;
1732
- let previousSeqNo = lastDurableSeqNo;
1733
- while (offset < data.length) {
1734
- if (offset + RECORD_LENGTH_SIZE > data.length) {
1735
- break;
1736
- }
1737
- const recordLength = view.getUint32(offset, false);
1738
- if (recordLength === 0 || recordLength > maxRecordBytes) {
1739
- break;
1740
- }
1741
- const payloadStart = offset + RECORD_LENGTH_SIZE;
1742
- const crcStart = payloadStart + recordLength;
1743
- const frameEnd = crcStart + FRAME_CRC_SIZE;
1744
- if (frameEnd > data.length) {
1745
- break;
1746
- }
1747
- const payload = data.subarray(payloadStart, crcStart);
1748
- if (crc32(payload) !== view.getUint32(crcStart, false)) {
1749
- break;
1750
- }
1751
- let entry;
1752
- try {
1753
- entry = decodeWalEntry(payload);
1754
- } catch {
1755
- break;
1756
- }
1757
- if (entry.seqNo <= previousSeqNo) {
1758
- break;
1759
- }
1760
- entries.push(entry);
1761
- previousSeqNo = entry.seqNo;
1762
- offset = frameEnd;
1763
- cleanEnd = frameEnd;
1764
- }
1765
- return { entries, cleanEnd };
1766
- }
1767
-
1768
- // src/persistence/durability/wal-segments.ts
1769
- function walPrefix(indexName, partitionId) {
1770
- return `${indexName}/wal/${partitionId}/`;
1771
- }
1772
- function nextSegmentKey(indexName, partitionId) {
1773
- return `${walPrefix(indexName, partitionId)}next`;
1774
- }
1775
- function segmentStartSeqNo(key, prefix) {
1776
- if (!key.startsWith(prefix)) {
1777
- return null;
1778
- }
1779
- const tail = key.slice(prefix.length);
1780
- if (!/^\d{16}$/.test(tail)) {
1781
- return null;
1782
- }
1783
- const value = Number.parseInt(tail, 10);
1784
- return Number.isSafeInteger(value) ? value : null;
1785
- }
1786
- async function collectSegments(directory, prefix) {
1787
- const keys = await directory.list(prefix);
1788
- const refs = [];
1789
- for (const key of keys) {
1790
- const startSeqNo = segmentStartSeqNo(key, prefix);
1791
- if (startSeqNo !== null) {
1792
- refs.push({ key, startSeqNo });
1793
- }
1794
- }
1795
- refs.sort((a, b) => a.startSeqNo - b.startSeqNo);
1796
- return refs;
1797
- }
1798
- async function openWal(directory, indexName, partitionId) {
1799
- const prefix = walPrefix(indexName, partitionId);
1800
- const markerBytes = await directory.read(`${prefix}commit`);
1801
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1802
- if (marker === null) {
1803
- return null;
1804
- }
1805
- return { marker, segments: await collectSegments(directory, prefix) };
1806
- }
1807
- async function* readSegmentsInOrder(directory, wal) {
1808
- const { marker, segments } = wal;
1809
- let highestRead = 0;
1810
- for (const { key, startSeqNo } of segments) {
1811
- if (startSeqNo > marker.state.activeSegmentSeqNo) {
1812
- continue;
1813
- }
1814
- const bytes = await directory.read(key);
1815
- if (bytes === null) {
1816
- continue;
1817
- }
1818
- if (startSeqNo < marker.state.activeSegmentSeqNo) {
1819
- const header = checkSegmentHeader(bytes);
1820
- if (!header.ok) {
1821
- throw new NarsilError(header.code, `Sealed WAL segment header invalid: ${header.reason}`, {
1822
- key,
1823
- reason: header.reason
1824
- });
1825
- }
1826
- const durableEntries2 = readDurableRegion(bytes, bytes.length);
1827
- for (const entry of durableEntries2) highestRead = Math.max(highestRead, entry.seqNo);
1828
- yield { durableEntries: durableEntries2, activeTail: null };
1829
- continue;
1830
- }
1831
- const durableEntries = readDurableRegion(bytes, marker.state.durableByteLength);
1832
- for (const entry of durableEntries) highestRead = Math.max(highestRead, entry.seqNo);
1833
- const tail = readTailBeyondFrontier(
1834
- bytes,
1835
- marker.state.durableByteLength,
1836
- Math.max(highestRead, marker.state.highestDurableSeqNo)
1837
- );
1838
- yield {
1839
- durableEntries,
1840
- activeTail: { key, entries: tail.entries, cleanEnd: tail.cleanEnd, segmentLength: bytes.length }
1841
- };
1842
- }
1843
- }
1844
- async function readWalSegments(directory, indexName, partitionId) {
1845
- const wal = await openWal(directory, indexName, partitionId);
1846
- if (wal === null) {
1847
- return null;
1848
- }
1849
- const durableEntries = [];
1850
- let highestReadFromWal = 0;
1851
- let activeTail = null;
1852
- for await (const segment of readSegmentsInOrder(directory, wal)) {
1853
- for (const entry of segment.durableEntries) {
1854
- durableEntries.push(entry);
1855
- highestReadFromWal = Math.max(highestReadFromWal, entry.seqNo);
1856
- }
1857
- activeTail = segment.activeTail ?? activeTail;
1858
- }
1859
- return { marker: wal.marker, segments: wal.segments, durableEntries, activeTail, highestReadFromWal };
1860
- }
1861
- function durableRecordMissing(indexName, partitionId, highestRead, marker) {
1862
- return new NarsilError(
1863
- ErrorCodes.PERSISTENCE_WAL_CORRUPT,
1864
- "A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
1865
- { indexName, partitionId, highestRead, highestDurable: marker.state.highestDurableSeqNo }
1866
- );
1867
- }
1868
- async function* walEntriesInRange(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive) {
1869
- const wal = await openWal(directory, indexName, partitionId);
1870
- if (wal === null) {
1871
- return;
1872
- }
1873
- let highestRead = 0;
1874
- for await (const segment of readSegmentsInOrder(directory, wal)) {
1875
- for (const entry of segment.durableEntries) {
1876
- highestRead = Math.max(highestRead, entry.seqNo);
1877
- if (entry.seqNo > fromSeqNoExclusive && entry.seqNo <= upToSeqNoInclusive) {
1878
- yield entry;
1879
- }
1880
- }
1881
- }
1882
- if (Math.max(fromSeqNoExclusive, highestRead) < wal.marker.state.highestDurableSeqNo) {
1883
- throw durableRecordMissing(indexName, partitionId, highestRead, wal.marker);
1884
- }
1885
- }
1886
- function writeRecordsOf(documents, written) {
1887
- const seqNos = new Float64Array(documents.length);
1888
- const primaryTerms = new Float64Array(documents.length);
1889
- for (let at = 0; at < documents.length; at++) {
1890
- const entry = written.get(documents[at].docId);
1891
- if (entry === void 0) continue;
1892
- seqNos[at] = entry.seqNo;
1893
- primaryTerms[at] = entry.primaryTerm;
1894
- }
1895
- return { seqNos, primaryTerms };
1896
- }
1897
- function everyVectorFieldIsTopLevel(vectorFieldPaths) {
1898
- for (const fieldPath of vectorFieldPaths) {
1899
- if (fieldPath.includes(".")) return false;
1900
- }
1901
- return vectorFieldPaths.size > 0;
1902
- }
1903
- function documentWithoutVectors(encoded, vectorFieldPaths, leavesVectorsUndecoded) {
1904
- const skipped = leavesVectorsUndecoded ? decodeMapWithoutFields(encoded, vectorFieldPaths) : null;
1905
- if (skipped !== null) return skipped;
1906
- const document = decode(encoded);
1907
- restoreVectorFields(document, vectorFieldPaths);
1908
- return prepareDocumentVectors(document, vectorFieldPaths).partitionDoc;
1909
- }
1910
- async function buildSegmentFromEntries(input) {
1911
- const leavesVectorsUndecoded = everyVectorFieldIsTopLevel(input.vectorFieldPaths);
1912
- const live = /* @__PURE__ */ new Map();
1913
- const deleted = /* @__PURE__ */ new Set();
1914
- const written = /* @__PURE__ */ new Map();
1915
- for await (const entry of input.entries) {
1916
- if (entry.operation === "DELETE") {
1917
- deleted.add(entry.documentId);
1918
- live.delete(entry.documentId);
1919
- written.delete(entry.documentId);
1920
- continue;
1921
- }
1922
- if (entry.document === null) continue;
1923
- live.delete(entry.documentId);
1924
- live.set(entry.documentId, documentWithoutVectors(entry.document, input.vectorFieldPaths, leavesVectorsUndecoded));
1925
- if (input.keepsWriteRecords) written.set(entry.documentId, entry);
1926
- }
1927
- const tombstones = [];
1928
- for (const docId of deleted) {
1929
- if (!live.has(docId)) tombstones.push(docId);
1930
- }
1931
- if (live.size === 0 && tombstones.length === 0) {
1932
- return null;
1933
- }
1934
- const documents = [];
1935
- for (const [docId, document] of live) documents.push({ docId, document });
1936
- const manager = createPartitionManager(input.indexName, input.config, input.language, createPartitionRouter(), 1);
1937
- const payload = buildSegmentPayload(
1938
- documents,
1939
- input.config.schema,
1940
- input.language,
1941
- resolvePartitionInsertOptions(input.config, manager.analysis),
1942
- input.config.trackPositions ?? true
1943
- );
1944
- const segment = createFrozenSegment(
1945
- payload,
1946
- documents.map((entry) => entry.document)
1947
- );
1948
- const built = { selections: [{ segment }], tombstones, docCount: live.size };
1949
- if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
1950
- return built;
1951
- }
1952
-
1953
- // src/persistence/durability/segment/compaction.ts
1954
- function windowFits(segments, start, windowLength, paged) {
1955
- let documents = 0;
1956
- let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
1957
- for (let at = start; at < start + windowLength; at++) {
1958
- documents += segments[at].docCount;
1959
- bytes += segments[at].bytes;
1960
- }
1961
- if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
1962
- return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
1963
- }
1964
- function smallestFittingWindow(segments, windowLength, paged) {
1965
- let best = null;
1966
- let bestBytes = Number.POSITIVE_INFINITY;
1967
- for (let start = 0; start + windowLength <= segments.length; start++) {
1968
- if (!windowFits(segments, start, windowLength, paged)) continue;
1969
- let bytes = 0;
1970
- for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
1971
- if (bytes < bestBytes) {
1972
- bestBytes = bytes;
1973
- best = { start, windowLength };
1974
- }
1975
- }
1976
- return best;
1977
- }
1978
- function chooseWindow(segments, wantedLength, paged) {
1979
- for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
1980
- const window = smallestFittingWindow(segments, windowLength, paged);
1981
- if (window !== null) return window;
1982
- }
1983
- return null;
1984
- }
1985
- function frozenOf(contents) {
1986
- return createSharedFrozenSegment(shareSegmentArrays(contents.arrays, null));
1987
- }
1988
- function applyTimeOrder(frozen, contents) {
1989
- for (let newer = 1; newer < frozen.length; newer++) {
1990
- const replaced = new Set(contents[newer].tombstones);
1991
- const ids = frozen[newer].arrays.idTable;
1992
- for (let ordinal = 0; ordinal < ids.count; ordinal++) replaced.add(ids.idAt(ordinal));
1993
- for (let older = 0; older < newer; older++) {
1994
- for (const docId of replaced) frozen[older].tombstoneDocument(docId);
1995
- }
1996
- }
1997
- }
1998
- function writeRecordsOf2(frozen, contents) {
1999
- if (contents.every((segment) => segment.writeRecords === void 0)) return void 0;
2000
- const records = /* @__PURE__ */ new Map();
2001
- for (let at = 0; at < frozen.length; at++) {
2002
- const held = contents[at].writeRecords;
2003
- if (held !== void 0) records.set(frozen[at], held);
2004
- }
2005
- return records;
2006
- }
2007
- function retainedTombstones(frozen, contents) {
2008
- const retained = /* @__PURE__ */ new Set();
2009
- for (const segment of contents) {
2010
- for (const docId of segment.tombstones) {
2011
- if (!frozen.some((candidate) => candidate.hasDocument(docId))) retained.add(docId);
2012
- }
2013
- }
2014
- return [...retained];
2015
- }
2016
- async function compactPartitionSegments(input) {
2017
- const { segments, compactionThreshold } = input;
2018
- if (segments.length <= compactionThreshold) {
2019
- return { segments, nextSegmentId: input.nextSegmentId };
2020
- }
2021
- const paged = input.paged === true;
2022
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
2023
- if (window === null) {
2024
- return { segments, nextSegmentId: input.nextSegmentId };
2025
- }
2026
- const windowRefs = segments.slice(window.start, window.start + window.windowLength);
2027
- const replacing = (written2) => ({
2028
- segments: [
2029
- ...segments.slice(0, window.start),
2030
- ...written2.segments,
2031
- ...segments.slice(window.start + window.windowLength)
2032
- ],
2033
- nextSegmentId: written2.nextSegmentId
2034
- });
2035
- if (paged) {
2036
- const { directory } = input.store;
2037
- if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
2038
- const { mergePagedWindow } = await import('./module-6RP4ZCBW.mjs');
2039
- return replacing(
2040
- await mergePagedWindow({
2041
- store: input.store,
2042
- directory,
2043
- indexName: input.indexName,
2044
- partitionId: input.partitionId,
2045
- schema: input.schema,
2046
- partialWordAnalysis: input.partialWordAnalysis,
2047
- textAnalysis: input.textAnalysis,
2048
- nextSegmentId: input.nextSegmentId,
2049
- refs: windowRefs,
2050
- dropsEveryTombstone: window.start === 0
2051
- })
2052
- );
2053
- }
2054
- const contents = [];
2055
- for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
2056
- const frozen = contents.map(frozenOf);
2057
- applyTimeOrder(frozen, contents);
2058
- const writeRecords = writeRecordsOf2(frozen, contents);
2059
- const written = await writeSegmentsOf({
2060
- store: input.store,
2061
- indexName: input.indexName,
2062
- partitionId: input.partitionId,
2063
- nextSegmentId: input.nextSegmentId,
2064
- schema: input.schema,
2065
- partialWordAnalysis: input.partialWordAnalysis,
2066
- textAnalysis: input.textAnalysis,
2067
- selections: frozen.map((segment) => ({ segment })),
2068
- tombstones: retainedTombstones(frozen, contents),
2069
- mergesIntoOneFile: true,
2070
- ...writeRecords === void 0 ? {} : { writeRecords }
2071
- });
2072
- return replacing(written);
2073
- }
2074
-
2075
- // src/persistence/durability/segment/write.ts
2076
- function textAnalysisOf(config) {
2077
- let resolved = null;
2078
- const resolve = () => {
2079
- resolved ??= {
2080
- language: getLanguage(config.language ?? "english"),
2081
- options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
2082
- };
2083
- return resolved;
2084
- };
2085
- return {
2086
- get language() {
2087
- return resolve().language;
2088
- },
2089
- get options() {
2090
- return resolve().options;
2091
- }
2092
- };
2093
- }
2094
- function partialWordAnalysisOf(config, text) {
2095
- const fields = partialFieldsOf(config.schema);
2096
- if (fields.length === 0) return null;
2097
- return {
2098
- fields,
2099
- get language() {
2100
- return text.language;
2101
- },
2102
- get options() {
2103
- return text.options;
2104
- }
2105
- };
2106
- }
2107
- function resolveCompactionThreshold(configured) {
2108
- if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
2109
- return DEFAULT_COMPACTION_THRESHOLD;
2110
- }
2111
- return configured;
2112
- }
2113
- function wholeContentsOf(input, partitionId) {
2114
- const whole = input.wholePartitions?.find((candidate) => candidate.partitionId === partitionId);
2115
- if (whole !== void 0) return whole;
2116
- const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
2117
- return captured === void 0 ? void 0 : contentsOfCaptured(captured);
2118
- }
2119
- function pagedSettingsOf(context) {
2120
- if (context.input.metadata.textStorage !== "disk") return {};
2121
- return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
2122
- }
2123
- async function tombstonesOfDelta(context, older, selections, deleted) {
2124
- const { directory } = context.input.store;
2125
- if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
2126
- const { pagedTombstonesFor } = await import('./module-6RP4ZCBW.mjs');
2127
- return pagedTombstonesFor(
2128
- directory,
2129
- older,
2130
- selections.map((selection) => selection.segment),
2131
- deleted
2132
- );
2133
- }
2134
- async function writeWholePartition(context, partitionId, priorPartition, whole) {
2135
- const written = await writeSegmentsOf({
2136
- store: context.input.store,
2137
- indexName: context.indexName,
2138
- partitionId,
2139
- nextSegmentId: priorPartition?.nextSegmentId ?? 0,
2140
- schema: context.input.metadata.schema,
2141
- partialWordAnalysis: context.partialWordAnalysis,
2142
- textAnalysis: context.textAnalysis,
2143
- selections: whole.selections,
2144
- tombstones: [],
2145
- ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
2146
- ...pagedSettingsOf(context)
2147
- });
2148
- return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
2149
- }
2150
- async function writeLoggedChanges(context, partitionId, priorPartition, entries) {
2151
- const built = await buildSegmentFromEntries({
2152
- indexName: context.indexName,
2153
- config: context.config,
2154
- language: getLanguage(context.config.language ?? "english"),
2155
- vectorFieldPaths: context.vectorFieldPaths,
2156
- entries,
2157
- keepsWriteRecords: context.input.keepsWriteRecords === true
2158
- });
2159
- let segments = priorPartition ? [...priorPartition.segments] : [];
2160
- let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
2161
- if (built !== null) {
2162
- const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
2163
- const written = await writeSegmentsOf({
2164
- store: context.input.store,
2165
- indexName: context.indexName,
2166
- partitionId,
2167
- nextSegmentId,
2168
- schema: context.input.metadata.schema,
2169
- partialWordAnalysis: context.partialWordAnalysis,
2170
- textAnalysis: context.textAnalysis,
2171
- selections: built.selections,
2172
- tombstones,
2173
- ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
2174
- ...pagedSettingsOf(context)
2175
- });
2176
- segments.push(...written.segments);
2177
- nextSegmentId = written.nextSegmentId;
2178
- }
2179
- if (segments.length > MAX_SEGMENTS_PER_PARTITION) {
2180
- throw new NarsilError(
2181
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2182
- `Partition ${partitionId} would exceed the maximum of ${MAX_SEGMENTS_PER_PARTITION} segments`,
2183
- { partitionId, segmentCount: segments.length, maximum: MAX_SEGMENTS_PER_PARTITION }
2184
- );
2185
- }
2186
- const compacted = await compactPartitionSegments({
2187
- store: context.input.store,
2188
- indexName: context.indexName,
2189
- partitionId,
2190
- schema: context.input.metadata.schema,
2191
- partialWordAnalysis: context.partialWordAnalysis,
2192
- textAnalysis: context.textAnalysis,
2193
- segments,
2194
- nextSegmentId,
2195
- compactionThreshold: context.compactionThreshold,
2196
- paged: context.input.metadata.textStorage === "disk"
2197
- });
2198
- segments = compacted.segments;
2199
- nextSegmentId = compacted.nextSegmentId;
2200
- return { partitionId, nextSegmentId, segments };
2201
- }
2202
- function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
2203
- if (prior === null) return;
2204
- const covered = new Set(targets.map((target) => target.partitionId));
2205
- for (const priorPartition of prior.partitions) {
2206
- if (covered.has(priorPartition.partitionId)) continue;
2207
- partitions.push(priorPartition);
2208
- const priorCheckpoint = prior.checkpoint.find((c) => c.partitionId === priorPartition.partitionId);
2209
- if (priorCheckpoint !== void 0 && !checkpointByPartition.has(priorPartition.partitionId)) {
2210
- checkpointByPartition.set(priorPartition.partitionId, priorCheckpoint);
2211
- }
2212
- }
2213
- }
2214
- async function writeCheckpointSegments(input) {
2215
- const config = reconstructSchemaFromMetadata(input.metadata);
2216
- const textAnalysis = textAnalysisOf(config);
2217
- const context = {
2218
- input,
2219
- indexName: input.metadata.indexName,
2220
- config,
2221
- partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
2222
- textAnalysis,
2223
- vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
2224
- compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
2225
- };
2226
- const checkpointByPartition = /* @__PURE__ */ new Map();
2227
- for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
2228
- const partitions = [];
2229
- for (const target of input.targets) {
2230
- const priorPartition = input.prior?.partitions.find((p) => p.partitionId === target.partitionId);
2231
- const whole = wholeContentsOf(input, target.partitionId);
2232
- if (whole !== void 0) {
2233
- partitions.push(await writeWholePartition(context, target.partitionId, priorPartition, whole));
2234
- continue;
2235
- }
2236
- if (input.directory === void 0) {
2237
- throw new NarsilError(
2238
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2239
- `The checkpoint of partition ${target.partitionId} holds neither the partition's content nor a log to read it from`,
2240
- { partitionId: target.partitionId }
2241
- );
2242
- }
2243
- const priorSeqNo = checkpointLastSeqNo(input.prior?.checkpoint ?? [], target.partitionId);
2244
- const entries = walEntriesInRange(
2245
- input.directory,
2246
- context.indexName,
2247
- target.partitionId,
2248
- priorSeqNo,
2249
- target.lastSeqNo
2250
- );
2251
- partitions.push(await writeLoggedChanges(context, target.partitionId, priorPartition, entries));
2252
- }
2253
- carryForwardUncheckpointedPartitions(input.prior, input.targets, partitions, checkpointByPartition);
2254
- return { checkpoint: [...checkpointByPartition.values()], partitions };
2255
- }
2256
- async function commitCheckpointManifest(store, metadata, written, vectors) {
2257
- const indexName = metadata.indexName;
2258
- const manifest = {
2259
- version: SEGMENT_MANIFEST_VERSION,
2260
- schema: metadata.schema,
2261
- language: metadata.language,
2262
- ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
2263
- checkpoint: written.checkpoint,
2264
- partitions: written.partitions,
2265
- vectors
2266
- };
2267
- const packed = await encodeSegmentManifest(manifest);
2268
- await store.write(manifestKey(indexName), packed.chunks);
2269
- return { garbage: await unreferencedKeys(store, indexName, manifest) };
2270
- }
2271
- async function unreferencedKeys(store, indexName, manifest) {
2272
- const referenced = manifestReferencedKeys(manifest);
2273
- const stored = await store.list(segmentsPrefix(indexName));
2274
- return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
2275
- }
2276
- async function removeCheckpointGarbage(store, garbage) {
2277
- for (const key of garbage) {
2278
- await store.remove(key);
2279
- }
2280
- }
2281
-
2282
- // src/persistence/durability/segment/load-partition.ts
2283
- function requireAgreedSchema(agreed, schema, key) {
2284
- for (const [field, type] of Object.entries(schema)) {
2285
- const existing = agreed[field];
2286
- if (existing !== void 0 && existing !== type) {
2287
- throw new NarsilError(
2288
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2289
- `The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
2290
- { key, field, existing, found: type }
2291
- );
2292
- }
2293
- }
2294
- }
2295
- function removeWhatTheSegmentReplaces(manager, segment, contents) {
2296
- manager.beginBatchRemove();
2297
- try {
2298
- for (const docId of contents.tombstones) {
2299
- if (manager.has(docId)) manager.remove(docId);
2300
- }
2301
- const ids = segment.arrays.idTable;
2302
- for (let ordinal = 0; ordinal < ids.count; ordinal++) {
2303
- const docId = ids.idAt(ordinal);
2304
- if (manager.has(docId)) manager.remove(docId);
2305
- }
2306
- } finally {
2307
- manager.endBatchRemove();
2308
- }
2309
- }
2310
- function yieldToEventLoop() {
2311
- return new Promise((resolve) => setTimeout(resolve, 0));
2312
- }
2313
- function removeFromPartition(manager, partitionId, docIds) {
2314
- const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
2315
- composite.beginBatch();
2316
- try {
2317
- for (const docId of docIds) {
2318
- if (!composite.has(docId)) continue;
2319
- composite.remove(docId, manager.config.schema, manager.language, insertOptions);
2320
- docPartitionMap.delete(docId);
2321
- }
2322
- } finally {
2323
- composite.endBatch();
2324
- }
2325
- }
2326
- async function attachPagedSegment(directory, ref, partitionId, manager, options) {
2327
- const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-6RP4ZCBW.mjs');
2328
- if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
2329
- const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
2330
- requireAgreedSchema(options.schema, snapshot.schema, ref.key);
2331
- const segment = createPagedFrozenSegment(snapshot);
2332
- removeFromPartition(manager, partitionId, snapshot.tombstones);
2333
- options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
2334
- if (snapshot.documentCount > 0) {
2335
- manager.attachFrozenSegment(partitionId, segment);
2336
- return segment;
2337
- }
2338
- segment.paged?.close();
2339
- return null;
2340
- }
2341
- function documentsHeldBefore(manager, partitionId) {
2342
- const held = manager.partitionAt(partitionId);
2343
- return held === void 0 ? [] : [...held.docIds()];
2344
- }
2345
- function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
2346
- const absent = heldBefore.filter(
2347
- (docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
2348
- );
2349
- if (absent.length === 0) return;
2350
- manager.beginBatchRemove();
2351
- try {
2352
- for (const docId of absent) manager.remove(docId);
2353
- } finally {
2354
- manager.endBatchRemove();
2355
- }
2356
- }
2357
- async function loadPartitionSegments(store, partition, manager, options) {
2358
- const heldBefore = documentsHeldBefore(manager, partition.partitionId);
2359
- const loaded = await attachPartitionSegments(store, partition, manager, options);
2360
- removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
2361
- }
2362
- async function attachPartitionSegments(store, partition, manager, options) {
2363
- const loaded = [];
2364
- const { directory } = store;
2365
- for (const ref of partition.segments) {
2366
- if (manager.config.textStorage === "disk" && directory !== void 0) {
2367
- const { isPagedSegmentFile } = await import('./module-6RP4ZCBW.mjs');
2368
- if (await isPagedSegmentFile(directory, ref.key)) {
2369
- const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
2370
- if (paged !== null) loaded.push(paged);
2371
- await yieldToEventLoop();
2372
- continue;
2373
- }
2374
- }
2375
- const contents = await readSegmentFile(store, ref);
2376
- requireAgreedSchema(options.schema, contents.schema, ref.key);
2377
- if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
2378
- const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
2379
- removeWhatTheSegmentReplaces(manager, segment, contents);
2380
- options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
2381
- if (contents.arrays.documentCount > 0) {
2382
- manager.attachFrozenSegment(partition.partitionId, segment);
2383
- loaded.push(segment);
2384
- }
2385
- await yieldToEventLoop();
2386
- }
2387
- return loaded;
2388
- }
2389
-
2390
- // src/persistence/durability/segment/load.ts
2391
- async function readSegmentManifest(store, indexName) {
2392
- const bytes = await store.read(manifestKey(indexName));
2393
- if (bytes === null) return null;
2394
- return decodeSegmentManifest(bytes, indexName);
2395
- }
2396
- async function readCurrentManifest(store, indexName) {
2397
- const decoded = await readSegmentManifest(store, indexName);
2398
- if (decoded === null) return null;
2399
- if (decoded.kind === "current") return decoded.manifest;
2400
- throw new NarsilError(
2401
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2402
- `Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
2403
- { indexName }
2404
- );
2405
- }
2406
- function growPartitionsTo(manager, highestPartitionId) {
2407
- while (manager.partitionCount <= highestPartitionId) {
2408
- const before = manager.partitionCount;
2409
- try {
2410
- manager.addPartition();
2411
- } catch {
2412
- break;
2413
- }
2414
- if (manager.partitionCount === before) break;
2415
- }
2416
- }
2417
- async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
2418
- let highestPartitionId = manifest.partitions.length - 1;
2419
- for (const partition of manifest.partitions) {
2420
- if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
2421
- }
2422
- growPartitionsTo(deps.manager, highestPartitionId);
2423
- const { directory } = store;
2424
- const opensVectorsOnDemand = manifest.vectors.some(
2425
- (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
2426
- );
2427
- if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
2428
- const { startReadingOpeningSectionsAhead } = await import('./module-6RP4ZCBW.mjs');
2429
- await startReadingOpeningSectionsAhead(directory, manifest);
2430
- }
2431
- for (const partition of manifest.partitions) {
2432
- if (partition.partitionId >= deps.manager.partitionCount) {
2433
- throw new NarsilError(
2434
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2435
- `Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
2436
- { indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
2437
- );
2438
- }
2439
- await loadPartitionSegments(store, partition, deps.manager, {
2440
- schema: manifest.schema,
2441
- decodesEveryDocument,
2442
- writeRecords: deps.writeRecords ?? null
2443
- });
2444
- }
2445
- for (const vector of manifest.vectors) {
2446
- const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
2447
- if (vectorIndex === void 0) continue;
2448
- await loadVectorField(
2449
- store,
2450
- vector,
2451
- vectorIndex,
2452
- decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
2453
- );
2454
- }
2455
- return manifest.checkpoint;
2456
- }
2457
- async function reclaimUnlistedKeys(store, indexName, referenced) {
2458
- for (const key of await store.list(segmentsPrefix(indexName))) {
2459
- if (!referenced.has(key)) await store.remove(key);
2460
- }
2461
- }
2462
- function validateHnswMetric2(value) {
2463
- if (value === void 0 || value === null) return void 0;
2464
- if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
2465
- return value;
2466
- }
2467
- return void 0;
2468
- }
2469
- function storedDocumentsOf(raw) {
2470
- requireValidFieldTypes(raw.schema ?? {});
2471
- const documents = {};
2472
- for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
2473
- documents[docId] = {
2474
- fields: doc.fields,
2475
- fieldLengths: doc.field_lengths ?? {}
2476
- };
2477
- }
2478
- dropStoredVectorValues(documents, raw.schema ?? {});
2479
- return documents;
2480
- }
2481
- function documentsOfPayloadV2(data) {
2482
- const raw = decode(data);
2483
- return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
2484
- }
2485
- function deserializePayloadV2(data) {
2486
- const raw = decode(data);
2487
- const documents = storedDocumentsOf(raw);
2488
- const fieldNames = raw.inverted_index?.field_names ?? [];
2489
- const invertedIndex = {};
2490
- for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
2491
- const count = list.ids.length;
2492
- const postings = new Array(count);
2493
- for (let i = 0; i < count; i++) {
2494
- postings[i] = {
2495
- docId: list.ids[i],
2496
- termFrequency: list.tf[i],
2497
- field: fieldNames[list.fi[i]] ?? "",
2498
- positions: list.pos ? list.pos[i] : []
2499
- };
2500
- }
2501
- invertedIndex[token] = {
2502
- docFrequency: list.df,
2503
- postings
2504
- };
2505
- }
2506
- const numeric = {};
2507
- for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
2508
- numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
2509
- }
2510
- const boolean = {};
2511
- for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
2512
- boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
2513
- }
2514
- const geopoint = {};
2515
- for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
2516
- geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
2517
- }
2518
- const vectorData = {};
2519
- for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
2520
- vectorData[field] = {
2521
- dimension: data2.dimension,
2522
- vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
2523
- hnswGraph: data2.hnsw_graph ? {
2524
- entryPoint: data2.hnsw_graph.entry_point,
2525
- maxLayer: data2.hnsw_graph.max_layer,
2526
- m: data2.hnsw_graph.m,
2527
- efConstruction: data2.hnsw_graph.ef_construction,
2528
- metric: validateHnswMetric2(data2.hnsw_graph.metric),
2529
- nodes: data2.hnsw_graph.nodes
2530
- } : null,
2531
- codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
2532
- };
2533
- }
2534
- return {
2535
- indexName: raw.index_name,
2536
- partitionId: raw.partition_id,
2537
- totalPartitions: raw.total_partitions,
2538
- language: raw.language ?? "english",
2539
- schema: raw.schema ?? {},
2540
- docCount: raw.doc_count ?? 0,
2541
- avgDocLength: raw.avg_doc_length ?? 0,
2542
- documents,
2543
- invertedIndex,
2544
- fieldIndexes: {
2545
- numeric,
2546
- boolean,
2547
- enum: raw.field_indexes?.enum ?? {},
2548
- geopoint,
2549
- ...patternIndexesFromWire(raw.field_indexes?.pattern)
2550
- },
2551
- surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
2552
- vectorData,
2553
- statistics: {
2554
- totalDocuments: raw.statistics?.total_documents ?? 0,
2555
- totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
2556
- averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
2557
- docFrequencies: raw.statistics?.doc_frequencies ?? {}
2558
- }
2559
- };
2560
- }
2561
-
2562
- export { FRAME_CRC_SIZE, MAX_WAL_RECORD_BYTES, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, buildEntry, captureWholePartition, checkSegmentHeader, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeWalEntry, deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, growPartitionsTo, indexMetadataOf, loadSegmentedCheckpoint, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, nextSegmentKey, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };