@delali/narsil 0.3.1-next.144 → 0.3.1-next.146

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/dist/adapters/filesystem.d.ts +1 -1
  2. package/dist/adapters/filesystem.mjs +101 -13
  3. package/dist/adapters/gcs.d.ts +49 -0
  4. package/dist/adapters/gcs.mjs +213 -0
  5. package/dist/adapters/indexeddb.d.ts +1 -1
  6. package/dist/adapters/indexeddb.mjs +2 -7
  7. package/dist/adapters/memory.d.ts +1 -1
  8. package/dist/adapters/memory.mjs +5 -0
  9. package/dist/adapters/s3.d.ts +53 -0
  10. package/dist/adapters/s3.mjs +263 -0
  11. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  12. package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
  13. package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
  14. package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-M65AZQQG.mjs} +8 -5
  15. package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-H3YIOLXZ.mjs} +1 -1
  16. package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-2JABMB4S.mjs} +10 -6
  17. package/dist/{chunk-46NZOPHS.mjs → browser/chunk-4DYLCICG.mjs} +2 -2
  18. package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-4RD3AFSB.mjs} +1 -1
  19. package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-5V74FFZE.mjs} +2 -2
  20. package/dist/{chunk-NN33CZRE.mjs → browser/chunk-6OWX3OP2.mjs} +2 -2
  21. package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
  22. package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-BVX2MEHA.mjs} +306 -607
  23. package/dist/browser/{chunk-KTOK275N.mjs → chunk-D6UFGBF5.mjs} +15 -45
  24. package/dist/browser/{chunk-D33A5VMU.mjs → chunk-IAXZAVYL.mjs} +25 -66
  25. package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
  26. package/dist/browser/{chunk-LVY2LI35.mjs → chunk-KAEVGLYN.mjs} +2 -2
  27. package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-LURXVCHG.mjs} +1 -1
  28. package/dist/browser/chunk-MPK76GVS.mjs +63 -0
  29. package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-NVEJJP4R.mjs} +1 -1
  30. package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-ODQP7P3H.mjs} +455 -66
  31. package/dist/browser/{chunk-35B42UFM.mjs → chunk-PIS7ZAP7.mjs} +12 -25
  32. package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-QMNZFFD3.mjs} +4 -3
  33. package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-SRL47USS.mjs} +2 -2
  34. package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
  35. package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-W643QV3L.mjs} +6 -5
  36. package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
  37. package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-XILIT5MT.mjs} +8 -8
  38. package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-A5S6JFDN.mjs} +20 -20
  39. package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-W5AVV3RF.mjs} +120 -80
  40. package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
  41. package/dist/browser/import-PTT63AGK.mjs +12 -0
  42. package/dist/browser/{module-BUFNZMEU.mjs → module-CEKA7MQH.mjs} +13 -10
  43. package/dist/browser/module-X5PGOE47.mjs +4288 -0
  44. package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
  45. package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
  46. package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
  47. package/dist/browser/{runtime-VEUH674B.mjs → runtime-D2ULZGCN.mjs} +3 -2
  48. package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-DQ6A4PQP.mjs} +20 -15
  49. package/dist/browser/{write-WHFBDWQS.mjs → write-MAGRAIEH.mjs} +14 -11
  50. package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-BT-HLDC8.d.ts} +3 -3
  51. package/dist/{chunk-O4LJQI4G.mjs → chunk-2HJWQPR6.mjs} +126 -4
  52. package/dist/{chunk-Z4HFMYNX.mjs → chunk-3BVGJ4JR.mjs} +36 -13
  53. package/dist/{chunk-ZPWS2SKJ.mjs → chunk-3HV7M5OJ.mjs} +272 -112
  54. package/dist/{chunk-UIKN2OTQ.mjs → chunk-3MXV3VHR.mjs} +2 -23
  55. package/dist/{browser/chunk-CCOAOO73.mjs → chunk-3SBJVQLL.mjs} +2 -2
  56. package/dist/chunk-3TVDTACC.mjs +104 -0
  57. package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
  58. package/dist/{chunk-SBMAWTRG.mjs → chunk-5WB2A3XU.mjs} +84 -15
  59. package/dist/{chunk-NPNK57CA.mjs → chunk-65WCV5JO.mjs} +1 -1
  60. package/dist/chunk-6BWLDLS3.mjs +58 -0
  61. package/dist/{chunk-UUZCQQG2.mjs → chunk-6C5EX2QR.mjs} +3 -3
  62. package/dist/{chunk-VEFN6X6Q.mjs → chunk-6F6USYII.mjs} +120 -13
  63. package/dist/{chunk-AST3SHFA.mjs → chunk-7HKWO4BJ.mjs} +7 -7
  64. package/dist/{chunk-W2COJ5AH.mjs → chunk-7N223GWV.mjs} +2 -2
  65. package/dist/{chunk-GWOS5DYC.mjs → chunk-ADBS5XRI.mjs} +11 -13
  66. package/dist/{chunk-TNBIK2LQ.mjs → chunk-AWNHDBKV.mjs} +430 -75
  67. package/dist/{chunk-ZMBFWQXN.mjs → chunk-B2FTZNNF.mjs} +2 -2
  68. package/dist/chunk-B7XIOAMK.mjs +100 -0
  69. package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
  70. package/dist/{chunk-QSK35UC2.mjs → chunk-BTPGQC6Z.mjs} +3 -11
  71. package/dist/{chunk-LA4WNDHL.mjs → chunk-CYYSZPVV.mjs} +12 -14
  72. package/dist/{chunk-SRJBFG2T.mjs → chunk-DOY6EG76.mjs} +6 -7
  73. package/dist/{chunk-6SNBQESJ.mjs → chunk-F22IOCR2.mjs} +10 -6
  74. package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
  75. package/dist/{chunk-L76W3IZJ.mjs → chunk-IRWP524D.mjs} +1 -1
  76. package/dist/{chunk-XS2VM7UE.mjs → chunk-IZT3G24Q.mjs} +2 -2
  77. package/dist/{chunk-NOT5QOHU.mjs → chunk-J7M73VY7.mjs} +1 -1
  78. package/dist/chunk-K5HSUPYS.mjs +11 -0
  79. package/dist/chunk-KAKQ74LS.mjs +53 -0
  80. package/dist/{chunk-ZSWHPKGM.mjs → chunk-KYVRZAYP.mjs} +69 -2
  81. package/dist/{chunk-NRUXBYPH.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
  82. package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-KZQIVGW7.mjs} +2 -2
  83. package/dist/{chunk-KZL72VTC.mjs → chunk-L7TIAYPX.mjs} +11 -10
  84. package/dist/{chunk-DD2673TX.mjs → chunk-LDGGR7YT.mjs} +4 -81
  85. package/dist/{chunk-XZ3SRNDJ.mjs → chunk-LJ4ENXOD.mjs} +77 -16
  86. package/dist/chunk-LQG5G2LY.mjs +39 -0
  87. package/dist/{chunk-HNHCNTVG.mjs → chunk-MDF6PDIA.mjs} +11 -0
  88. package/dist/{chunk-74XB2SXI.mjs → chunk-NVTFQFUL.mjs} +16 -2
  89. package/dist/{chunk-BOJZGUS6.mjs → chunk-NYSONUB7.mjs} +1 -1
  90. package/dist/{chunk-WJGO5PGV.mjs → chunk-O4KUNDVF.mjs} +31 -17
  91. package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
  92. package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
  93. package/dist/{chunk-NJJCPFXN.mjs → chunk-QEYF2SQ6.mjs} +2 -2
  94. package/dist/{chunk-7AUASRE7.mjs → chunk-QN5WVV5K.mjs} +7 -7
  95. package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
  96. package/dist/chunk-SOUZMPME.mjs +1635 -0
  97. package/dist/{chunk-C3YDYX6C.mjs → chunk-T4CJDBBE.mjs} +1 -1
  98. package/dist/{chunk-YLNSJQLX.mjs → chunk-TTCKT2RT.mjs} +5 -5
  99. package/dist/{chunk-KEIYVZUX.mjs → chunk-U2726TL5.mjs} +5 -5
  100. package/dist/{chunk-RKAVHNRW.mjs → chunk-U46IZNCW.mjs} +10 -5
  101. package/dist/{chunk-BRUUBXCB.mjs → chunk-U6NE6RCV.mjs} +10 -38
  102. package/dist/{chunk-VJEGOELZ.mjs → chunk-VP6XRMWR.mjs} +4 -4
  103. package/dist/{chunk-IO5F7ZBE.mjs → chunk-VQDZVWN3.mjs} +874 -413
  104. package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
  105. package/dist/{chunk-YHUTF6EA.mjs → chunk-XKPNKR3V.mjs} +17 -47
  106. package/dist/{chunk-HJJWY6PT.mjs → chunk-YGQOPB2Z.mjs} +1 -1
  107. package/dist/chunk-YKEHQUSZ.mjs +248 -0
  108. package/dist/{chunk-GLZHNSGB.mjs → chunk-ZAYWOHVG.mjs} +1 -1
  109. package/dist/{chunk-ICW45TTU.mjs → chunk-ZEKDO7YX.mjs} +2 -2
  110. package/dist/chunk-ZXNNR73R.mjs +31 -0
  111. package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
  112. package/dist/{chunk-RHKRTYGO.mjs → chunk-ZYKUD5NX.mjs} +9 -7
  113. package/dist/{client-B12YFALP.d.ts → client-Cn3gNxUU.d.ts} +62 -5
  114. package/dist/client.d.ts +8 -8
  115. package/dist/client.mjs +46 -0
  116. package/dist/{cold-module-4X7WJCFM.mjs → cold-module-2WWGBFJC.mjs} +43 -49
  117. package/dist/{config-CIG5yDyg.d.ts → config-CfhH1IWQ.d.ts} +150 -3
  118. package/dist/{config-BhZxIeb1.d.ts → config-aIsy7mcL.d.ts} +63 -3
  119. package/dist/direct-executor-QLVHMELJ.mjs +24 -0
  120. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  121. package/dist/distribution/coordinator/etcd.mjs +1 -1
  122. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  123. package/dist/distribution/transport/grpc.mjs +4 -4
  124. package/dist/distribution/transport/in-memory.mjs +2 -2
  125. package/dist/distribution/transport/tcp.mjs +3 -3
  126. package/dist/distribution.d.ts +8 -8
  127. package/dist/distribution.mjs +16116 -14537
  128. package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-GNWQSZI3.mjs} +142 -109
  129. package/dist/embeddings/openai.d.ts +1 -1
  130. package/dist/embeddings/openai.mjs +2 -2
  131. package/dist/{engine-CtzbHOmG.d.ts → engine-CJRUHXgf.d.ts} +3 -3
  132. package/dist/gzip-F4HMIEEQ.mjs +1 -0
  133. package/dist/import-DX55PN4G.mjs +41 -0
  134. package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
  135. package/dist/index.browser.mjs +7 -4
  136. package/dist/index.d.ts +542 -8
  137. package/dist/index.mjs +29 -32
  138. package/dist/index.react-native.mjs +19 -0
  139. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  140. package/dist/invalidation/filesystem.d.ts +1 -1
  141. package/dist/invalidation/filesystem.mjs +1 -1
  142. package/dist/invalidation/noop.d.ts +1 -1
  143. package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-SCUE62TO.mjs} +23 -25
  144. package/dist/{module-DXSJ5LT2.mjs → module-HW6IXUM6.mjs} +603 -112
  145. package/dist/module-V5K4ZP23.mjs +45 -0
  146. package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-FDIIWZNR.mjs} +42 -41
  147. package/dist/partial-words.mjs +7 -9
  148. package/dist/pattern.mjs +4 -6
  149. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
  150. package/dist/persistence/durability/checkpoint-worker.mjs +36 -39
  151. package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
  152. package/dist/pool-UPV5PVPY.mjs +298 -0
  153. package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
  154. package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
  155. package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
  156. package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
  157. package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
  158. package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
  159. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
  160. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
  161. package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
  162. package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
  163. package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
  164. package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
  165. package/dist/{browser/chunk-6RHTAVZX.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +38 -15
  166. package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
  167. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
  168. package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
  169. package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
  170. package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
  171. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
  172. package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
  173. package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
  174. package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
  175. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
  176. package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
  177. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
  178. package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
  179. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
  180. package/dist/{chunk-KGOXHLLZ.mjs → react-native/chunks/chunk-PU55IOG4.mjs} +228 -11
  181. package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
  182. package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
  183. package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
  184. package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
  185. package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
  186. package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
  187. package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
  188. package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
  189. package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
  190. package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
  191. package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
  192. package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
  193. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
  194. package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
  195. package/dist/{browser/module-BVP4TKZU.mjs → react-native/chunks/module-JBBH6PZH.mjs} +671 -146
  196. package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
  197. package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
  198. package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
  199. package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
  200. package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
  201. package/dist/{browser/restore-queue-ONYNYUDE.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
  202. package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
  203. package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
  204. package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
  205. package/dist/{chunk-DQNKVOJ2.mjs → react-native/partial-words.mjs} +13 -228
  206. package/dist/react-native/pattern.mjs +2442 -0
  207. package/dist/react-native/synonyms.mjs +460 -0
  208. package/dist/react.d.ts +11 -11
  209. package/dist/react.mjs +11 -0
  210. package/dist/{rerank-KTLC3E6N.mjs → rerank-3W54N6P2.mjs} +4 -4
  211. package/dist/rerankers/http.d.ts +1 -1
  212. package/dist/rerankers/http.mjs +2 -2
  213. package/dist/restore-queue-BQWWNNQI.mjs +41 -0
  214. package/dist/{runtime-DYV75DCB.mjs → runtime-L2DZYY6N.mjs} +5 -5
  215. package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
  216. package/dist/server.d.ts +24 -12
  217. package/dist/server.mjs +159 -38
  218. package/dist/{snapshot-CGM65VCD.mjs → snapshot-EHXTOTW5.mjs} +47 -50
  219. package/dist/synonyms.mjs +3 -4
  220. package/dist/{tasks-CjRJRGTb.d.ts → tasks-C4ElXr4P.d.ts} +59 -7
  221. package/dist/thread-I7DVHHW7.mjs +31 -0
  222. package/dist/vector/search-worker.d.ts +2 -2
  223. package/dist/vector/search-worker.mjs +8 -7
  224. package/dist/workers/entry.mjs +9 -9
  225. package/dist/{write-2DZPEQL2.mjs → write-SDJXXQ2G.mjs} +36 -39
  226. package/openapi.json +349 -2
  227. package/package.json +108 -8
  228. package/dist/analysis-change-HQUT2Z3I.mjs +0 -43
  229. package/dist/browser/import-TLB2A5LJ.mjs +0 -9
  230. package/dist/chunk-5WYVUPGD.mjs +0 -427
  231. package/dist/chunk-AV4IJKD2.mjs +0 -35
  232. package/dist/chunk-GES7U2PM.mjs +0 -68
  233. package/dist/chunk-N37VMC62.mjs +0 -18
  234. package/dist/chunk-WNBRW4Q2.mjs +0 -128
  235. package/dist/chunk-WQBCBKY3.mjs +0 -8
  236. package/dist/chunk-ZATRDAYU.mjs +0 -160
  237. package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
  238. package/dist/import-OOBUW3RR.mjs +0 -44
  239. package/dist/module-IYPLWHGC.mjs +0 -48
  240. package/dist/thread-KGNAIKPD.mjs +0 -33
  241. /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
@@ -0,0 +1,1205 @@
1
+ import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-QC4ZRBSX.mjs';
2
+ import { vectorsPerFile } from './chunk-GV5KFQ6M.mjs';
3
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-FKGG4UOF.mjs';
4
+ import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-5FV4ZEVA.mjs';
5
+ import './chunk-HURV3ASY.mjs';
6
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-LQ6Q4L5B.mjs';
7
+ import './chunk-LRGWK367.mjs';
8
+ import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-B6OD63Z3.mjs';
9
+ import './chunk-7VWQLEFV.mjs';
10
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration } from './chunk-YJSMEKRE.mjs';
11
+ import './chunk-LQAKO5QV.mjs';
12
+ import './chunk-NNE4VDMM.mjs';
13
+ import './chunk-L4M7QJNP.mjs';
14
+ import './chunk-IO2SMUQD.mjs';
15
+ import './chunk-AM7RSBPF.mjs';
16
+ import './chunk-ZCKNBPQ4.mjs';
17
+ import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, parseVectorDimension, flattenSchema } from './chunk-PU55IOG4.mjs';
18
+ import { MAX_FILE_BYTES } from './chunk-TJMSKBEM.mjs';
19
+ import './chunk-DKJTZ625.mjs';
20
+ import { getLanguage, resolveIndexAnalysis } from './chunk-3CTWSQRW.mjs';
21
+ import './chunk-2J2727V4.mjs';
22
+ import './chunk-KWRW5U65.mjs';
23
+ import './chunk-PPEPTUVV.mjs';
24
+ import './chunk-LVIKKAA6.mjs';
25
+ import './chunk-AAAETEJK.mjs';
26
+ import './chunk-JV6VQY2S.mjs';
27
+ import './chunk-NFGBOI46.mjs';
28
+ import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
29
+ import { decode } from '@msgpack/msgpack';
30
+
31
+ // src/persistence/archive/memory-store.ts
32
+ function joined(chunks) {
33
+ let total = 0;
34
+ for (const chunk of chunks) total += chunk.byteLength;
35
+ const bytes = new Uint8Array(total);
36
+ let offset = 0;
37
+ for (const chunk of chunks) {
38
+ bytes.set(chunk, offset);
39
+ offset += chunk.byteLength;
40
+ }
41
+ return bytes;
42
+ }
43
+ function memoryCheckpointStore() {
44
+ const files = /* @__PURE__ */ new Map();
45
+ return {
46
+ files,
47
+ read: async (key) => files.get(key) ?? null,
48
+ write: async (key, chunks) => {
49
+ files.set(key, joined(chunks));
50
+ },
51
+ remove: async (key) => {
52
+ files.delete(key);
53
+ },
54
+ list: async (prefix) => [...files.keys()].filter((key) => key.startsWith(prefix)).sort(),
55
+ byteLength: async (key) => files.get(key)?.byteLength ?? null
56
+ };
57
+ }
58
+
59
+ // src/persistence/archive/from-memory.ts
60
+ function frozenPlan(plan) {
61
+ const files = [];
62
+ for (let index = 0; index < plan.newFiles; index++) files.push(plan.readNewFile(index));
63
+ return { ...plan, readNewFile: (index) => files[index] };
64
+ }
65
+ function partitionVectorPlan(vectorIndex, docIds) {
66
+ const held = [];
67
+ const vectors = [];
68
+ for (const docId of docIds) {
69
+ const vector = vectorIndex.getVector(docId);
70
+ if (vector === null) continue;
71
+ held.push(docId);
72
+ vectors.push(Float32Array.from(vector));
73
+ }
74
+ const capacity = vectorsPerFile(vectorIndex.dimension, null);
75
+ const files = [];
76
+ for (let start = 0; start < held.length; start += capacity) {
77
+ const slice = held.slice(start, start + capacity);
78
+ const packed = new Float32Array(slice.length * vectorIndex.dimension);
79
+ for (let i = 0; i < slice.length; i++) packed.set(vectors[start + i], i * vectorIndex.dimension);
80
+ const payload = {
81
+ v: VECTOR_FILE_PAYLOAD_VERSION,
82
+ fieldName: vectorIndex.fieldName,
83
+ dimension: vectorIndex.dimension,
84
+ docIds: slice,
85
+ codes: null,
86
+ vectors: vectorsToBytes(packed)
87
+ };
88
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
89
+ }
90
+ return {
91
+ unchanged: false,
92
+ storesHeads: false,
93
+ kept: [],
94
+ keptFiles: [],
95
+ newFiles: files.length,
96
+ graphParts: null,
97
+ signature: { revision: 0, graphSize: -1, calibration: null },
98
+ readNewFile: (index) => files[index],
99
+ release: () => void 0
100
+ };
101
+ }
102
+ function capture(input) {
103
+ const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
104
+ const { indexName } = input;
105
+ const writeRecords = input.writeRecords ?? null;
106
+ const targets = partitionIds.map((partitionId) => {
107
+ const target = {
108
+ partitionId,
109
+ lastSeqNo: input.appliedSeqNo(partitionId),
110
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
111
+ };
112
+ if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
113
+ return target;
114
+ });
115
+ const contents = partitionIds.map(
116
+ (partitionId) => partitionContentsOf(
117
+ input.manager,
118
+ partitionId,
119
+ writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
120
+ )
121
+ );
122
+ const plans = /* @__PURE__ */ new Map();
123
+ const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
124
+ for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
125
+ plans.set(
126
+ fieldPath,
127
+ partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
128
+ );
129
+ }
130
+ return { targets, contents, plans };
131
+ }
132
+ async function buildArchiveFromMemory(input) {
133
+ if (input.partitionId === null) {
134
+ for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
135
+ }
136
+ const captured = capture(input);
137
+ const store = memoryCheckpointStore();
138
+ const written = await writeCheckpointSegments({
139
+ store,
140
+ metadata: input.metadata,
141
+ targets: captured.targets,
142
+ compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
143
+ prior: null,
144
+ wholePartitions: captured.contents,
145
+ keepsWriteRecords: input.writeRecords != null
146
+ });
147
+ const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
148
+ const refs = vectors.refs;
149
+ await commitCheckpointManifest(store, input.metadata, written, refs);
150
+ const manifestBytes = store.files.get(manifestKey(input.indexName));
151
+ const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName);
152
+ if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
153
+ throw new NarsilError(
154
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
155
+ `The archive of index "${input.indexName}" holds no manifest`
156
+ );
157
+ }
158
+ return {
159
+ manifestBytes,
160
+ manifest: decoded.manifest,
161
+ readFile: async (ref) => {
162
+ const bytes = store.files.get(ref.key);
163
+ if (bytes === void 0) {
164
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The archive file "${ref.key}" is missing`, {
165
+ key: ref.key
166
+ });
167
+ }
168
+ return bytes;
169
+ },
170
+ readRange: async (ref, offset, length) => {
171
+ const bytes = store.files.get(ref.key);
172
+ if (bytes === void 0 || offset + length > bytes.byteLength) {
173
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive file "${ref.key}" is missing`, {
174
+ key: ref.key
175
+ });
176
+ }
177
+ return bytes.subarray(offset, offset + length);
178
+ },
179
+ release: () => store.files.clear()
180
+ };
181
+ }
182
+
183
+ // src/persistence/archive/constants.ts
184
+ var MAX_PAX_HEADER_BYTES = 65536;
185
+ var ARCHIVE_FILE_CHUNK_BYTES = 4194304;
186
+
187
+ // src/persistence/archive/tar.ts
188
+ var TAR_BLOCK_BYTES = 512;
189
+ var NAME_OFFSET = 0;
190
+ var NAME_BYTES = 100;
191
+ var MODE_OFFSET = 100;
192
+ var OWNER_OFFSET = 108;
193
+ var GROUP_OFFSET = 116;
194
+ var ID_FIELD_BYTES = 8;
195
+ var SIZE_OFFSET = 124;
196
+ var SIZE_FIELD_BYTES = 12;
197
+ var SIZE_FIELD_DIGITS = 11;
198
+ var MTIME_OFFSET = 136;
199
+ var MTIME_FIELD_BYTES = 12;
200
+ var CHECKSUM_FIELD_OFFSET = 148;
201
+ var CHECKSUM_FIELD_BYTES = 8;
202
+ var CHECKSUM_DIGITS = 6;
203
+ var TYPEFLAG_OFFSET = 156;
204
+ var MAGIC_OFFSET = 257;
205
+ var MAGIC_BYTES = 6;
206
+ var VERSION_OFFSET = 263;
207
+ var VERSION_BYTES = 2;
208
+ var PREFIX_OFFSET = 345;
209
+ var PREFIX_BYTES = 155;
210
+ var MODE_OCTAL = "0000644";
211
+ var ZERO_OCTAL_7 = "0000000";
212
+ var ZERO_OCTAL_11 = "00000000000";
213
+ var REGULAR_FILE = 48;
214
+ var OLD_REGULAR_FILE = 0;
215
+ var PAX_EXTENDED_HEADER = 120;
216
+ var PAX_HEADER_NAME = "PaxHeader";
217
+ var encoder = new TextEncoder();
218
+ var decoder = new TextDecoder("utf-8", { fatal: true });
219
+ function writeString(block, offset, length, value) {
220
+ const bytes = encoder.encode(value);
221
+ block.set(bytes.subarray(0, length), offset);
222
+ }
223
+ function writeOctal(block, offset, digits, value) {
224
+ writeString(block, offset, digits, value.toString(8).padStart(digits, "0"));
225
+ }
226
+ function headerChecksum(block) {
227
+ let sum = 0;
228
+ const checksumEnd = CHECKSUM_FIELD_OFFSET + CHECKSUM_FIELD_BYTES;
229
+ for (let at = 0; at < TAR_BLOCK_BYTES; at++) {
230
+ sum += at >= CHECKSUM_FIELD_OFFSET && at < checksumEnd ? 32 : block[at];
231
+ }
232
+ return sum;
233
+ }
234
+ function ustarBlock(name, prefix, size, typeflag) {
235
+ const block = new Uint8Array(TAR_BLOCK_BYTES);
236
+ writeString(block, NAME_OFFSET, NAME_BYTES, name);
237
+ writeString(block, MODE_OFFSET, ID_FIELD_BYTES, MODE_OCTAL);
238
+ writeString(block, OWNER_OFFSET, ID_FIELD_BYTES, ZERO_OCTAL_7);
239
+ writeString(block, GROUP_OFFSET, ID_FIELD_BYTES, ZERO_OCTAL_7);
240
+ writeOctal(block, SIZE_OFFSET, SIZE_FIELD_DIGITS, size);
241
+ writeString(block, MTIME_OFFSET, MTIME_FIELD_BYTES, ZERO_OCTAL_11);
242
+ block[TYPEFLAG_OFFSET] = typeflag;
243
+ writeString(block, MAGIC_OFFSET, MAGIC_BYTES, "ustar");
244
+ writeString(block, VERSION_OFFSET, VERSION_BYTES, "00");
245
+ writeString(block, PREFIX_OFFSET, PREFIX_BYTES, prefix);
246
+ const checksum = headerChecksum(block);
247
+ writeString(block, CHECKSUM_FIELD_OFFSET, CHECKSUM_DIGITS, checksum.toString(8).padStart(CHECKSUM_DIGITS, "0"));
248
+ block[CHECKSUM_FIELD_OFFSET + CHECKSUM_DIGITS] = 0;
249
+ block[CHECKSUM_FIELD_OFFSET + CHECKSUM_DIGITS + 1] = 32;
250
+ return block;
251
+ }
252
+ function splitForUstar(name) {
253
+ if (encoder.encode(name).byteLength <= NAME_BYTES) return { name, prefix: "" };
254
+ for (let slash = name.indexOf("/"); slash >= 0; slash = name.indexOf("/", slash + 1)) {
255
+ const prefix = name.slice(0, slash);
256
+ const rest = name.slice(slash + 1);
257
+ if (encoder.encode(prefix).byteLength <= PREFIX_BYTES && encoder.encode(rest).byteLength <= NAME_BYTES) {
258
+ return { name: rest, prefix };
259
+ }
260
+ }
261
+ return null;
262
+ }
263
+ function paxRecord(key, value) {
264
+ const body = ` ${key}=${value}
265
+ `;
266
+ const bodyBytes = encoder.encode(body).byteLength;
267
+ let digits = 1;
268
+ while (String(bodyBytes + digits).length !== digits) digits += 1;
269
+ return `${bodyBytes + digits}${body}`;
270
+ }
271
+ function tarPadding(size) {
272
+ const remainder = size % TAR_BLOCK_BYTES;
273
+ return new Uint8Array(remainder === 0 ? 0 : TAR_BLOCK_BYTES - remainder);
274
+ }
275
+ function tarEntryHeaders(name, size) {
276
+ const split = splitForUstar(name);
277
+ if (split !== null) return [ustarBlock(split.name, split.prefix, size, REGULAR_FILE)];
278
+ const record = encoder.encode(paxRecord("path", name));
279
+ return [
280
+ ustarBlock(PAX_HEADER_NAME, "", record.byteLength, PAX_EXTENDED_HEADER),
281
+ record,
282
+ tarPadding(record.byteLength),
283
+ ustarBlock(name.slice(-NAME_BYTES).replace(/^[^/]*\//, ""), "", size, REGULAR_FILE)
284
+ ];
285
+ }
286
+ function tarEnd() {
287
+ return new Uint8Array(TAR_BLOCK_BYTES * 2);
288
+ }
289
+ function invalidArchive(message, details) {
290
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid snapshot archive: ${message}`, details);
291
+ }
292
+ function readString(block, offset, length) {
293
+ let end = offset;
294
+ while (end < offset + length && block[end] !== 0) end++;
295
+ try {
296
+ return decoder.decode(block.subarray(offset, end));
297
+ } catch {
298
+ invalidArchive("an entry header holds a name that is not UTF-8");
299
+ }
300
+ }
301
+ function readOctal(block, offset, length, what) {
302
+ const text = readString(block, offset, length).trim();
303
+ if (!/^[0-7]+$/.test(text)) invalidArchive(`an entry header holds an invalid ${what}`);
304
+ const value = Number.parseInt(text, 8);
305
+ if (!Number.isSafeInteger(value)) invalidArchive(`an entry header holds an invalid ${what}`);
306
+ return value;
307
+ }
308
+ function isZeroBlock(block) {
309
+ for (let at = 0; at < block.length; at++) if (block[at] !== 0) return false;
310
+ return true;
311
+ }
312
+ function parseTarHeader(block) {
313
+ if (readOctal(block, CHECKSUM_FIELD_OFFSET, CHECKSUM_FIELD_BYTES, "checksum") !== headerChecksum(block)) {
314
+ invalidArchive("an entry header fails its checksum");
315
+ }
316
+ const typeflag = block[TYPEFLAG_OFFSET];
317
+ if (typeflag !== REGULAR_FILE && typeflag !== OLD_REGULAR_FILE && typeflag !== PAX_EXTENDED_HEADER) {
318
+ invalidArchive(`an entry has type ${String.fromCharCode(typeflag)}, while an archive holds regular files alone`);
319
+ }
320
+ const name = readString(block, NAME_OFFSET, NAME_BYTES);
321
+ const prefix = readString(block, PREFIX_OFFSET, PREFIX_BYTES);
322
+ return {
323
+ name: prefix.length > 0 ? `${prefix}/${name}` : name,
324
+ size: readOctal(block, SIZE_OFFSET, SIZE_FIELD_BYTES, "size"),
325
+ isPaxHeader: typeflag === PAX_EXTENDED_HEADER
326
+ };
327
+ }
328
+ var SPACE = 32;
329
+ var EQUALS = 61;
330
+ var NEWLINE = 10;
331
+ function decodeRecordText(bytes) {
332
+ try {
333
+ return decoder.decode(bytes);
334
+ } catch {
335
+ invalidArchive("a pax extended header is not UTF-8");
336
+ }
337
+ }
338
+ function parsePaxRecords(body) {
339
+ const records = /* @__PURE__ */ new Map();
340
+ let at = 0;
341
+ while (at < body.length) {
342
+ const space = body.indexOf(SPACE, at);
343
+ if (space < 0) invalidArchive("a pax extended header holds a malformed record");
344
+ const length = Number.parseInt(decodeRecordText(body.subarray(at, space)), 10);
345
+ if (!Number.isSafeInteger(length) || length <= space - at || at + length > body.length) {
346
+ invalidArchive("a pax extended header holds a malformed record");
347
+ }
348
+ const record = body.subarray(space + 1, at + length);
349
+ const equals = record.indexOf(EQUALS);
350
+ if (equals < 0 || record[record.length - 1] !== NEWLINE) {
351
+ invalidArchive("a pax extended header holds a record without a value");
352
+ }
353
+ records.set(decodeRecordText(record.subarray(0, equals)), decodeRecordText(record.subarray(equals + 1, -1)));
354
+ at += length;
355
+ }
356
+ return records;
357
+ }
358
+
359
+ // src/persistence/archive/read.ts
360
+ var ARCHIVE_CONFIG_ENTRY = "config";
361
+ var ARCHIVE_MANIFEST_ENTRY = "manifest";
362
+ var SEGMENTS_MARKER = "/segments/";
363
+ var CHECKSUM_OFFSET = 14;
364
+ async function readBlock(reader) {
365
+ const block = await reader.readExactly(TAR_BLOCK_BYTES);
366
+ if (block === null) return null;
367
+ if (block.byteLength < TAR_BLOCK_BYTES) invalidArchive("the archive ends inside an entry header");
368
+ return block;
369
+ }
370
+ async function skipPadding(reader, size) {
371
+ const remainder = size % TAR_BLOCK_BYTES;
372
+ if (remainder === 0) return;
373
+ const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
374
+ if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
375
+ invalidArchive("the archive ends inside the padding of an entry");
376
+ }
377
+ }
378
+ async function readPadded(reader, size) {
379
+ const body = await reader.readExactly(size) ?? new Uint8Array(0);
380
+ if (body.byteLength < size) invalidArchive("the archive ends inside an entry");
381
+ await skipPadding(reader, size);
382
+ return body;
383
+ }
384
+ function entryBody(reader, size) {
385
+ let remaining = size;
386
+ async function* chunks() {
387
+ while (remaining > 0) {
388
+ const length = Math.min(ARCHIVE_FILE_CHUNK_BYTES, remaining);
389
+ const chunk = await reader.readExactly(length);
390
+ if (chunk === null || chunk.byteLength < length) invalidArchive("the archive ends inside an entry");
391
+ remaining -= length;
392
+ yield chunk;
393
+ }
394
+ }
395
+ return chunks();
396
+ }
397
+ async function nextEntryHeader(reader) {
398
+ const block = await readBlock(reader);
399
+ if (block === null || isZeroBlock(block)) return null;
400
+ const header = parseTarHeader(block);
401
+ if (!header.isPaxHeader) return { name: header.name, size: header.size };
402
+ if (header.size > MAX_PAX_HEADER_BYTES) invalidArchive("a pax extended header is too long");
403
+ const records = parsePaxRecords(await readPadded(reader, header.size));
404
+ const next = await readBlock(reader);
405
+ if (next === null || isZeroBlock(next)) invalidArchive("a pax extended header precedes no entry");
406
+ const entry = parseTarHeader(next);
407
+ if (entry.isPaxHeader) invalidArchive("a pax extended header follows another");
408
+ const path = records.get("path");
409
+ const size = records.get("size");
410
+ const parsedSize = size === void 0 ? entry.size : Number.parseInt(size, 10);
411
+ if (!Number.isSafeInteger(parsedSize) || parsedSize < 0) invalidArchive("a pax extended header holds an invalid size");
412
+ return { name: path ?? entry.name, size: parsedSize };
413
+ }
414
+ async function readNamedEntry(reader, expected) {
415
+ const header = await nextEntryHeader(reader);
416
+ if (header === null) invalidArchive(`the archive ends before its ${expected} entry`);
417
+ if (header.name !== expected) invalidArchive(`the archive holds "${header.name}" where its ${expected} entry belongs`);
418
+ if (header.size > MAX_FILE_BYTES) invalidArchive(`the ${expected} entry passes the file size limit`);
419
+ return readPadded(reader, header.size);
420
+ }
421
+ function sourceIndexNameOf(keys) {
422
+ const first = keys[0];
423
+ if (first === void 0) return "archive";
424
+ const at = first.indexOf(SEGMENTS_MARKER);
425
+ if (at <= 0) invalidArchive(`the manifest lists the key "${first}" outside any index`);
426
+ return first.slice(0, at);
427
+ }
428
+ function headerChecksumOf(bytes) {
429
+ if (bytes.byteLength < HEADER_SIZE) return -1;
430
+ return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(CHECKSUM_OFFSET, false);
431
+ }
432
+ async function readSnapshotArchive(reader, handlers) {
433
+ await handlers.config(await readNamedEntry(reader, ARCHIVE_CONFIG_ENTRY));
434
+ const raw = await openManifestPayload(await readNamedEntry(reader, ARCHIVE_MANIFEST_ENTRY));
435
+ const sourceIndexName = sourceIndexNameOf(listedKeysOf(raw));
436
+ const decoded = decodeManifestPayload(raw, sourceIndexName);
437
+ if (decoded.kind !== "current") invalidArchive("the manifest holds the layout of an earlier release");
438
+ const manifest = decoded.manifest;
439
+ await handlers.manifest(manifest, sourceIndexName);
440
+ const files = manifestFileRefs(manifest);
441
+ const prefix = `${sourceIndexName}/`;
442
+ const listedNames = new Set(files.map((file) => file.key.slice(prefix.length)));
443
+ const seen = /* @__PURE__ */ new Set();
444
+ for (const ref of files) {
445
+ const expected = ref.key.slice(prefix.length);
446
+ const header = await nextEntryHeader(reader);
447
+ if (header === null) invalidArchive(`the archive ends before it holds "${expected}"`);
448
+ if (header.name !== expected) {
449
+ const why = !listedNames.has(header.name) ? "that the manifest does not list" : seen.has(header.name) ? "twice" : "out of order";
450
+ invalidArchive(`the archive holds the entry "${header.name}" ${why}`, { entry: header.name });
451
+ }
452
+ if (header.size !== ref.bytes) {
453
+ invalidArchive(`the entry "${expected}" holds ${header.size} bytes where the manifest lists ${ref.bytes}`);
454
+ }
455
+ const start = await reader.peek(Math.min(HEADER_SIZE, header.size));
456
+ if (headerChecksumOf(start) !== ref.checksum) {
457
+ invalidArchive(`the entry "${expected}" carries a checksum other than the manifest lists`, { entry: expected });
458
+ }
459
+ seen.add(expected);
460
+ const entry = entryBody(reader, header.size);
461
+ await handlers.file(expected, ref, entry);
462
+ let unread = await entry.next();
463
+ while (unread.done !== true) unread = await entry.next();
464
+ await skipPadding(reader, header.size);
465
+ }
466
+ const trailing = await nextEntryHeader(reader);
467
+ if (trailing !== null) {
468
+ invalidArchive(`the archive holds the entry "${trailing.name}" that the manifest does not list`, {
469
+ entry: trailing.name
470
+ });
471
+ }
472
+ }
473
+
474
+ // src/persistence/archive/write.ts
475
+ function entryChunks(name, bytes) {
476
+ return [...tarEntryHeaders(name, bytes.byteLength), bytes, tarPadding(bytes.byteLength)].filter(
477
+ (chunk) => chunk.byteLength > 0
478
+ );
479
+ }
480
+ async function* fileEntryChunks(source, name, ref) {
481
+ yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
482
+ for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
483
+ yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
484
+ }
485
+ const padding = tarPadding(ref.bytes);
486
+ if (padding.byteLength > 0) yield padding;
487
+ }
488
+ async function* archiveChunks(source) {
489
+ yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
490
+ yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
491
+ const prefix = `${source.sourceIndexName}/`;
492
+ for (const ref of manifestFileRefs(source.manifest)) {
493
+ yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
494
+ }
495
+ yield tarEnd();
496
+ }
497
+ function archiveStream(source) {
498
+ const chunks = archiveChunks(source);
499
+ let released = false;
500
+ const release = () => {
501
+ if (released) return;
502
+ released = true;
503
+ source.release();
504
+ };
505
+ return new ReadableStream({
506
+ async pull(controller) {
507
+ try {
508
+ const step = await chunks.next();
509
+ if (step.done) {
510
+ release();
511
+ controller.close();
512
+ return;
513
+ }
514
+ controller.enqueue(step.value);
515
+ } catch (error) {
516
+ release();
517
+ controller.error(error);
518
+ }
519
+ },
520
+ async cancel() {
521
+ release();
522
+ await chunks.return(void 0);
523
+ }
524
+ });
525
+ }
526
+
527
+ // src/persistence/archive/file-entry.ts
528
+ async function writeArchiveFile(store, key, chunks) {
529
+ const { directory } = store;
530
+ if (directory === void 0) {
531
+ const held = [];
532
+ for await (const chunk of chunks) held.push(chunk);
533
+ await store.write(key, held);
534
+ return;
535
+ }
536
+ const receiving = `${key}.receiving`;
537
+ await directory.remove(receiving);
538
+ const handle = await directory.appendHandle(receiving);
539
+ try {
540
+ for await (const chunk of chunks) await handle.append(chunk);
541
+ await handle.sync();
542
+ } finally {
543
+ await handle.close();
544
+ }
545
+ await directory.move(receiving, key);
546
+ }
547
+
548
+ // src/persistence/archive/validate.ts
549
+ function requireDimension(key, found, expected) {
550
+ if (expected === found) return;
551
+ throw new NarsilError(
552
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
553
+ `The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
554
+ { key, found, expected }
555
+ );
556
+ }
557
+ function requireSchemaOfManifest(key, schema, manifest) {
558
+ for (const [field, type] of Object.entries(schema)) {
559
+ const listed = manifest.schema[field];
560
+ if (listed === void 0 || listed === type) continue;
561
+ throw new NarsilError(
562
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
563
+ `The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
564
+ { key, field }
565
+ );
566
+ }
567
+ }
568
+ async function validateStagedCheckpoint(store, manifest) {
569
+ for (const partition of manifest.partitions) {
570
+ for (const ref of partition.segments) {
571
+ const { directory } = store;
572
+ if (directory !== void 0) {
573
+ const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-VEK2KCKU.mjs');
574
+ if (await isPagedSegmentFile(directory, ref.key)) {
575
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
576
+ continue;
577
+ }
578
+ }
579
+ const contents = await readSegmentFile(store, ref);
580
+ requireSchemaOfManifest(ref.key, contents.schema, manifest);
581
+ decodeEveryStoredDocument(contents.arrays.documentTable);
582
+ }
583
+ }
584
+ for (const field of manifest.vectors) {
585
+ const fieldType = manifest.schema[field.fieldPath];
586
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
587
+ for (const file of field.files) {
588
+ const { payload } = await readVectorFile(store, file, field.fieldPath);
589
+ requireDimension(file.key, payload.dimension, dimension);
590
+ }
591
+ await readGraphParts(store, field);
592
+ }
593
+ }
594
+
595
+ // src/workers/detached-index.ts
596
+ function attachablePolicy(shared) {
597
+ let attached = false;
598
+ const policy = {
599
+ get enabled() {
600
+ return attached && shared.enabled;
601
+ },
602
+ get count() {
603
+ return shared.count;
604
+ },
605
+ get host() {
606
+ return shared.host;
607
+ }
608
+ };
609
+ return {
610
+ policy,
611
+ attach() {
612
+ attached = true;
613
+ }
614
+ };
615
+ }
616
+ function createDetachedIndex(indexName, config, language, synonyms, executor) {
617
+ const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
618
+ const entry = executor.createDetachedEntry(indexName, config, language, policy);
619
+ entry.manager.synonyms = synonyms;
620
+ return { indexName, manager: entry.manager, entry, attach };
621
+ }
622
+
623
+ // src/engine/snapshot/restore-setup.ts
624
+ function prepareRestoredIndex(configuration, deps, reject) {
625
+ const configFields = restoredConfigFields(configuration, reject);
626
+ const embedding = restoredEmbedding(configuration.embedding, reject);
627
+ const languageName = configuration.language ?? "";
628
+ const language = getLanguage(languageName);
629
+ const schema = validateSchema(configuration.schema);
630
+ validatePatternValueLimit(configFields.patternValueLimit);
631
+ requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
632
+ validateVectorPromotion(configFields.vectorPromotion);
633
+ validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
634
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
635
+ if (configFields.required !== void 0 && configFields.required.length > 0) {
636
+ validateRequiredFieldsInSchema(configFields.required, schema);
637
+ }
638
+ const config = {
639
+ schema,
640
+ language: languageName,
641
+ ...configFields,
642
+ ...embedding !== void 0 ? {
643
+ embedding: {
644
+ fields: embedding.fields,
645
+ ...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
646
+ }
647
+ } : {}
648
+ };
649
+ resolveIndexAnalysis(config);
650
+ const adapterName = embedding?.adapter ?? null;
651
+ let embeddingAdapter = null;
652
+ if (embedding !== void 0) {
653
+ const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
654
+ if (candidate) {
655
+ validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
656
+ embeddingAdapter = candidate;
657
+ }
658
+ }
659
+ const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
660
+ const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
661
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
662
+ return {
663
+ config,
664
+ language,
665
+ embeddingAdapter,
666
+ adapterName,
667
+ storedRevision,
668
+ analysisGeneration,
669
+ storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
670
+ synonyms
671
+ };
672
+ }
673
+ function restoredSynonyms(raw, config, language, reject) {
674
+ if (raw === void 0 || raw === null) return null;
675
+ return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
676
+ }
677
+ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
678
+ const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
679
+ if (loadingVectorCode !== null) await loadingVectorCode;
680
+ const detached = createDetachedIndex(
681
+ indexName,
682
+ prepared.config,
683
+ prepared.language,
684
+ prepared.synonyms?.lookup ?? null,
685
+ deps.executor
686
+ );
687
+ while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
688
+ return detached;
689
+ }
690
+ function restoredRegistryEntry(prepared, manager, deps) {
691
+ return {
692
+ config: prepared.config,
693
+ language: prepared.language,
694
+ embeddingAdapter: prepared.embeddingAdapter,
695
+ embeddingAdapterName: prepared.adapterName,
696
+ vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
697
+ indexUuid: null,
698
+ heldPartitions: null,
699
+ documentCount: manager.countDocuments(),
700
+ partitionCount: manager.partitionCount,
701
+ analysisRevision: prepared.storedRevision,
702
+ arrayPositionGap: prepared.storedArrayPositionGap,
703
+ synonyms: prepared.synonyms?.encoded ?? null,
704
+ analysisGeneration: prepared.analysisGeneration,
705
+ pendingAnalysis: null,
706
+ durableAnalysis: null
707
+ };
708
+ }
709
+
710
+ // src/engine/snapshot/replace.ts
711
+ function releaseTimedOut(indexName, timeoutMs) {
712
+ return new NarsilError(
713
+ ErrorCodes.WORKER_TIMEOUT,
714
+ `The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
715
+ { indexName, timeoutMs }
716
+ );
717
+ }
718
+ async function withinLimit(indexName, release, work, deps) {
719
+ release.settled = work.catch(() => {
720
+ release.reloads = true;
721
+ });
722
+ let timer;
723
+ const limit = new Promise((_, reject) => {
724
+ timer = setTimeout(() => {
725
+ release.timedOut = true;
726
+ reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
727
+ }, deps.releaseTimeoutMs);
728
+ });
729
+ try {
730
+ await Promise.race([work, limit]);
731
+ } finally {
732
+ clearTimeout(timer);
733
+ }
734
+ }
735
+ function vectorIndexesOf(indexName, deps) {
736
+ const manager = deps.executor.getManager(indexName);
737
+ return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
738
+ }
739
+ async function holdVectorCopies(vectorIndexes, release) {
740
+ for (const vectorIndex of vectorIndexes) {
741
+ if (release.held.includes(vectorIndex)) continue;
742
+ release.held.push(vectorIndex);
743
+ await vectorIndex.holdWorkerCopies();
744
+ }
745
+ }
746
+ async function releaseCopies(indexName, release, deps) {
747
+ release.reloads = await deps.copies.releaseIndexCopies(indexName);
748
+ await holdVectorCopies(vectorIndexesOf(indexName, deps), release);
749
+ }
750
+ async function endRelease(indexName, release, deps) {
751
+ await release.settled;
752
+ for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
753
+ await deps.copies.endCopyRelease(indexName, release.reloads);
754
+ }
755
+ function persistedCheckpoint(indexName, partitionCount, deps) {
756
+ const durable = deps.durability?.manager;
757
+ if (durable === void 0) return [];
758
+ return Array.from({ length: partitionCount }, (_, partitionId) => ({
759
+ partitionId,
760
+ lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
761
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
762
+ }));
763
+ }
764
+ async function commitToStorage(indexName, restored, previousPartitions, deps) {
765
+ const durable = deps.durability?.manager;
766
+ const staging = restored.staging;
767
+ if (durable === void 0 || staging === null) return;
768
+ const previous = persistedCheckpoint(indexName, previousPartitions, deps);
769
+ if (previousPartitions > 0) await durable.unloadIndex(indexName);
770
+ const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
771
+ try {
772
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
773
+ await staging.commit();
774
+ } catch (error) {
775
+ if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
776
+ throw error;
777
+ }
778
+ await staging.finish();
779
+ const relocate = await staging.relocation().catch(() => null);
780
+ if (relocate === null) return;
781
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
782
+ const { manager } = restored.detached;
783
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
784
+ const partition = manager.getPartition(partitionId);
785
+ if (!isCompositePartition(partition)) continue;
786
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
787
+ }
788
+ }
789
+ function activate(indexName, restored, entry, deps) {
790
+ deps.executor.installDetachedIndex(restored.detached);
791
+ deps.indexRegistry.set(indexName, entry);
792
+ restored.detached.attach();
793
+ deps.clearAnalysisStale(indexName);
794
+ const { prepared } = restored;
795
+ if (prepared.storedRevision !== prepared.language.revision || arrayPositionsAreStale(prepared.config, prepared.storedArrayPositionGap)) {
796
+ deps.markAnalysisStale({
797
+ indexName,
798
+ language: prepared.language.name,
799
+ storedRevision: prepared.storedRevision,
800
+ currentRevision: prepared.language.revision
801
+ });
802
+ }
803
+ if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
804
+ deps.forgetWatermark(indexName);
805
+ }
806
+ async function replaceExisting(indexName, restored, entry, deps) {
807
+ const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
808
+ try {
809
+ await deps.abandonAnalysisChange(indexName);
810
+ await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
811
+ await deps.indexState.replace(indexName, async (wasOpen) => {
812
+ await deps.abandonAnalysisRebuild(indexName);
813
+ const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
814
+ const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf(indexName, deps);
815
+ await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
816
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
817
+ await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
818
+ activate(indexName, restored, entry, deps);
819
+ });
820
+ } finally {
821
+ const ending = endRelease(indexName, release, deps).catch(() => void 0);
822
+ if (!release.timedOut) await ending;
823
+ }
824
+ }
825
+ async function createFresh(indexName, restored, entry, deps) {
826
+ deps.copies.desyncIndex(indexName);
827
+ deps.executor.installDetachedIndex(restored.detached);
828
+ deps.indexRegistry.set(indexName, entry);
829
+ try {
830
+ await deps.indexState.registerReplacing(indexName, async () => {
831
+ await commitToStorage(indexName, restored, 0, deps);
832
+ activate(indexName, restored, entry, deps);
833
+ });
834
+ } catch (error) {
835
+ if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
836
+ throw error;
837
+ } finally {
838
+ await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
839
+ }
840
+ }
841
+ async function stageMetadata(indexName, staging, entry) {
842
+ const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
843
+ await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
844
+ }
845
+ async function installRestoredIndex(indexName, restored, deps) {
846
+ const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
847
+ let replaced;
848
+ try {
849
+ if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
850
+ replaced = deps.indexRegistry.get(indexName);
851
+ if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
852
+ else await createFresh(indexName, restored, entry, deps);
853
+ } catch (error) {
854
+ if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
855
+ else deps.executor.discardDetachedIndex(restored.detached);
856
+ throw error;
857
+ }
858
+ await restored.staging?.removeRetired().catch(() => void 0);
859
+ const durable = deps.durability?.manager;
860
+ if (restored.staging === null && durable !== void 0) {
861
+ await durable.persistMetadata(indexName);
862
+ await durable.checkpoint(indexName);
863
+ }
864
+ if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
865
+ await deps.announceIndexCreated(indexName, restored.prepared.config);
866
+ for (const vectorIndex of restored.detached.manager.getVectorIndexes().values()) {
867
+ vectorIndex.startShuffledRebuildWhenDue();
868
+ }
869
+ }
870
+
871
+ // src/engine/snapshot/archive-restore.ts
872
+ function rejectConfiguration(message) {
873
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
874
+ }
875
+ function renamed(manifest, sourceName, indexName) {
876
+ const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
877
+ return {
878
+ ...manifest,
879
+ partitions: manifest.partitions.map((partition) => ({
880
+ ...partition,
881
+ segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
882
+ })),
883
+ vectors: manifest.vectors.map((field) => ({
884
+ ...field,
885
+ files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
886
+ graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
887
+ }))
888
+ };
889
+ }
890
+ function requireSameSchema(prepared, manifest) {
891
+ const configured = flattenSchema(prepared.config.schema);
892
+ const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
893
+ for (const field of keys) {
894
+ if (configured[field] === manifest.schema[field]) continue;
895
+ throw new NarsilError(
896
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
897
+ `The archive's configuration gives field "${field}" another type than its manifest does`,
898
+ { field }
899
+ );
900
+ }
901
+ }
902
+ function partitionCountOf(manifest) {
903
+ let count = 1;
904
+ for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
905
+ return count;
906
+ }
907
+ function missingPart(what) {
908
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
909
+ }
910
+ function keepsTextOnDisk(prepared, staging) {
911
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
912
+ }
913
+ async function stageArchive(indexName, reader, staging, deps) {
914
+ const staged = { prepared: null, manifest: null };
915
+ await readSnapshotArchive(reader, {
916
+ config: async (bytes) => {
917
+ staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
918
+ },
919
+ manifest: async (manifest2, sourceName) => {
920
+ if (staged.prepared === null) missingPart("configuration");
921
+ requireSameSchema(staged.prepared, manifest2);
922
+ const replacesIndex = deps.indexRegistry.has(indexName);
923
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
924
+ requireRestoreMemory(
925
+ indexName,
926
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
927
+ replacesIndex
928
+ );
929
+ staged.manifest = renamed(manifest2, sourceName, indexName);
930
+ },
931
+ file: async (entryName, _ref, chunks) => {
932
+ if (staged.manifest === null) missingPart("manifest");
933
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
934
+ }
935
+ });
936
+ const { prepared, manifest } = staged;
937
+ if (prepared === null) missingPart("configuration");
938
+ if (manifest === null) missingPart("manifest");
939
+ const { directory } = staging;
940
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
941
+ const { bytesThatSegmentsHoldInMemory } = await import('./module-VEK2KCKU.mjs');
942
+ const replacesIndex = deps.indexRegistry.has(indexName);
943
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
944
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
945
+ }
946
+ await validateStagedCheckpoint(staging, manifest);
947
+ return { prepared, manifest };
948
+ }
949
+ async function loadStagedIndex(indexName, staged, staging, store, deps) {
950
+ const { prepared, manifest } = staged;
951
+ await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
952
+ const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
953
+ try {
954
+ await loadSegmentedCheckpoint(
955
+ store,
956
+ indexName,
957
+ manifest,
958
+ { manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
959
+ true
960
+ );
961
+ } catch (error) {
962
+ deps.executor.discardDetachedIndex(detached);
963
+ throw error;
964
+ }
965
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint };
966
+ }
967
+ async function restoreArchive(indexName, reader, deps) {
968
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
969
+ const store = staging?.store ?? memoryCheckpointStore();
970
+ try {
971
+ const staged = await stageArchive(indexName, reader, store, deps);
972
+ await installRestoredIndex(indexName, await loadStagedIndex(indexName, staged, staging, store, deps), deps);
973
+ } finally {
974
+ await staging?.discard().catch(() => void 0);
975
+ await reader.close();
976
+ }
977
+ }
978
+ function invalidSnapshot(message, details) {
979
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
980
+ }
981
+ async function decodeEarlierSnapshot(data) {
982
+ const payloadBytes = await unpackIndexSnapshotEnvelope(data);
983
+ let decoded;
984
+ try {
985
+ decoded = decode(payloadBytes);
986
+ } catch (err) {
987
+ invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
988
+ bytes: data.length
989
+ });
990
+ }
991
+ if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
992
+ invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
993
+ }
994
+ const envelope = decoded;
995
+ if (envelope.version !== 1 && envelope.version !== 2) {
996
+ invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
997
+ version: envelope.version
998
+ });
999
+ }
1000
+ if (!envelope.schema || typeof envelope.schema !== "object")
1001
+ invalidSnapshot("Invalid snapshot: missing or invalid schema");
1002
+ if (!envelope.language || typeof envelope.language !== "string") {
1003
+ invalidSnapshot("Invalid snapshot: missing or invalid language");
1004
+ }
1005
+ if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
1006
+ return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
1007
+ }
1008
+ function vectorsOfSnapshot(envelope) {
1009
+ const byDocument = /* @__PURE__ */ new Map();
1010
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
1011
+ for (const part of decodeVectorIndexParts(rawParts)) {
1012
+ const values = bytesToVectors(part.vectors);
1013
+ for (let i = 0; i < part.docIds.length; i++) {
1014
+ let fields = byDocument.get(part.docIds[i]);
1015
+ if (fields === void 0) {
1016
+ fields = /* @__PURE__ */ new Map();
1017
+ byDocument.set(part.docIds[i], fields);
1018
+ }
1019
+ fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
1020
+ }
1021
+ }
1022
+ }
1023
+ return byDocument;
1024
+ }
1025
+ function footprintOf(envelope, prepared, replacesIndex) {
1026
+ let partitionBytes = 0;
1027
+ for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
1028
+ const vectorCounts = /* @__PURE__ */ new Map();
1029
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
1030
+ let count = 0;
1031
+ for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
1032
+ vectorCounts.set(fieldPath, count);
1033
+ }
1034
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
1035
+ }
1036
+ function documentsOfPartition(envelope, payload) {
1037
+ return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
1038
+ }
1039
+ function insertRestoredDocument(manager, docId, fields, vectors) {
1040
+ manager.insert(docId, fields, { skipClone: true });
1041
+ if (vectors === void 0) return;
1042
+ try {
1043
+ insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
1044
+ } catch (err) {
1045
+ manager.remove(docId);
1046
+ throw err;
1047
+ }
1048
+ }
1049
+ function insertSnapshotDocuments(envelope, manager) {
1050
+ const vectors = vectorsOfSnapshot(envelope);
1051
+ for (const payload of envelope.partitions ?? []) {
1052
+ for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
1053
+ insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
1054
+ }
1055
+ }
1056
+ for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
1057
+ }
1058
+ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
1059
+ const entry = restoredRegistryEntry(prepared, built.manager, deps);
1060
+ const archive = await buildArchiveFromMemory({
1061
+ indexName,
1062
+ metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
1063
+ manager: built.manager,
1064
+ vectorIndexes: built.manager.getVectorIndexes(),
1065
+ appliedSeqNo: () => 0,
1066
+ partitionId: null
1067
+ });
1068
+ try {
1069
+ for (const ref of manifestFileRefs(archive.manifest))
1070
+ await staging.store.write(ref.key, [await archive.readFile(ref)]);
1071
+ } finally {
1072
+ archive.release();
1073
+ }
1074
+ await validateStagedCheckpoint(staging.store, archive.manifest);
1075
+ return archive.manifest;
1076
+ }
1077
+ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
1078
+ const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
1079
+ let manifest;
1080
+ try {
1081
+ insertSnapshotDocuments(envelope, built.manager);
1082
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
1083
+ manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
1084
+ } catch (err) {
1085
+ deps.executor.discardDetachedIndex(built);
1086
+ throw err;
1087
+ }
1088
+ deps.executor.discardDetachedIndex(built);
1089
+ return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
1090
+ }
1091
+ async function restoreEarlierSnapshot(indexName, data, deps) {
1092
+ const envelope = await decodeEarlierSnapshot(data);
1093
+ const prepared = prepareRestoredIndex(envelope, deps);
1094
+ const replacesIndex = deps.indexRegistry.has(indexName);
1095
+ requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
1096
+ const analysedNow = {
1097
+ ...prepared,
1098
+ storedRevision: prepared.language.revision,
1099
+ storedArrayPositionGap: ARRAY_POSITION_GAP
1100
+ };
1101
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
1102
+ try {
1103
+ await installRestoredIndex(
1104
+ indexName,
1105
+ await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
1106
+ deps
1107
+ );
1108
+ } finally {
1109
+ await staging?.discard().catch(() => void 0);
1110
+ }
1111
+ }
1112
+
1113
+ // src/engine/snapshot/index.ts
1114
+ var NRSL_MAGIC_BYTES = 4;
1115
+ async function checkpointForArchive(deps) {
1116
+ const durable = deps.durability?.manager;
1117
+ if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
1118
+ return buildArchiveFromMemory({
1119
+ indexName: deps.indexName,
1120
+ metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
1121
+ manager: deps.manager,
1122
+ vectorIndexes: deps.manager.getVectorIndexes(),
1123
+ appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
1124
+ partitionId: null
1125
+ });
1126
+ }
1127
+ async function createSnapshot(deps) {
1128
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
1129
+ const checkpoint = await checkpointForArchive(deps);
1130
+ try {
1131
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
1132
+ const config = await encodeIndexConfiguration(deps.entry);
1133
+ return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
1134
+ }
1135
+ } catch (error) {
1136
+ checkpoint.release();
1137
+ throw error;
1138
+ }
1139
+ checkpoint.release();
1140
+ }
1141
+ throw new NarsilError(
1142
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
1143
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
1144
+ { indexName: deps.indexName }
1145
+ );
1146
+ }
1147
+ function requireSnapshotSource(data) {
1148
+ if (data instanceof Uint8Array) return data;
1149
+ if (data !== null && typeof data === "object") {
1150
+ if (typeof data.getReader === "function") return data;
1151
+ if (typeof data[Symbol.asyncIterator] === "function") {
1152
+ return data;
1153
+ }
1154
+ }
1155
+ throw new NarsilError(
1156
+ ErrorCodes.DOC_VALIDATION_FAILED,
1157
+ "Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
1158
+ );
1159
+ }
1160
+ async function readEarlierSnapshotFile(reader) {
1161
+ const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
1162
+ if (length > MAX_FILE_BYTES) {
1163
+ throw new NarsilError(
1164
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1165
+ `The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
1166
+ { bytes: length }
1167
+ );
1168
+ }
1169
+ const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
1170
+ if (bytes.byteLength < length) {
1171
+ throw new NarsilError(
1172
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1173
+ "The snapshot ends before the payload its header declares"
1174
+ );
1175
+ }
1176
+ const trailing = await reader.readExactly(1);
1177
+ if (trailing !== null && trailing.byteLength > 0) {
1178
+ throw new NarsilError(
1179
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1180
+ "The snapshot holds bytes after the payload its header declares"
1181
+ );
1182
+ }
1183
+ return bytes;
1184
+ }
1185
+ async function restoreFromSnapshot(indexName, data, deps) {
1186
+ const reader = createByteReader(requireSnapshotSource(data));
1187
+ const head = await reader.peek(NRSL_MAGIC_BYTES);
1188
+ if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
1189
+ let bytes;
1190
+ try {
1191
+ bytes = await readEarlierSnapshotFile(reader);
1192
+ } finally {
1193
+ await reader.close();
1194
+ }
1195
+ await restoreEarlierSnapshot(indexName, bytes, deps);
1196
+ return;
1197
+ }
1198
+ if (head.byteLength === 0) {
1199
+ await reader.close();
1200
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
1201
+ }
1202
+ await restoreArchive(indexName, reader, deps);
1203
+ }
1204
+
1205
+ export { checkpointForArchive, createSnapshot, restoreFromSnapshot };