@delali/narsil 0.3.1-next.157 → 0.3.1-next.161

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (565) hide show
  1. package/dist/adapters/filesystem.mjs +3 -3
  2. package/dist/adapters/gcs.mjs +3 -3
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/adapters/s3.mjs +3 -3
  5. package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
  6. package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
  7. package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
  8. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  9. package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
  10. package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
  11. package/dist/browser/embed-RF2MPGAO.mjs +222 -0
  12. package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
  13. package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
  14. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  15. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  16. package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
  17. package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
  18. package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
  19. package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
  20. package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
  21. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  22. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  23. package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
  24. package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
  25. package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
  26. package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
  27. package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
  28. package/dist/chunk-2JRERUNZ.mjs +217 -0
  29. package/dist/chunk-2RIS76RJ.mjs +212 -0
  30. package/dist/chunk-3B775F3C.mjs +252 -0
  31. package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
  32. package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
  33. package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
  34. package/dist/chunk-6EZW2GQL.mjs +978 -0
  35. package/dist/chunk-75UNINFA.mjs +271 -0
  36. package/dist/chunk-77AUVDQ5.mjs +7 -0
  37. package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
  38. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  39. package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
  40. package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
  41. package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
  42. package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
  43. package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
  44. package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
  45. package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
  46. package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
  47. package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
  48. package/dist/chunk-GXCDWLQI.mjs +193 -0
  49. package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
  50. package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
  51. package/dist/chunk-IB4BYLE3.mjs +127 -0
  52. package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
  53. package/dist/chunk-IH3G6SJO.mjs +40 -0
  54. package/dist/chunk-JV74RKCY.mjs +3041 -0
  55. package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
  56. package/dist/chunk-KLVMK3R5.mjs +47 -0
  57. package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
  58. package/dist/chunk-LF4GB4PO.mjs +281 -0
  59. package/dist/chunk-NC4QLCZC.mjs +59 -0
  60. package/dist/chunk-OOO54OVC.mjs +627 -0
  61. package/dist/chunk-OUWZKNO3.mjs +207 -0
  62. package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
  63. package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
  64. package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
  65. package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
  66. package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
  67. package/dist/chunk-S2POFTLN.mjs +10 -0
  68. package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
  69. package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
  70. package/dist/chunk-SXIRM4MX.mjs +3549 -0
  71. package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
  72. package/dist/chunk-TINW4BSH.mjs +31301 -0
  73. package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
  74. package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
  75. package/dist/chunk-U6B7DUQW.mjs +788 -0
  76. package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
  77. package/dist/chunk-W34VJXY7.mjs +4723 -0
  78. package/dist/chunk-YETKPX5M.mjs +5 -0
  79. package/dist/chunk-ZQ6QY42L.mjs +59 -0
  80. package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
  81. package/dist/client.d.ts +8 -8
  82. package/dist/client.mjs +66 -18
  83. package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
  84. package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
  85. package/dist/core-leaf-KXFGP74A.mjs +7 -0
  86. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  87. package/dist/direct-executor-MKTZBR26.mjs +16 -0
  88. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  89. package/dist/distribution/coordinator/etcd.mjs +7 -4
  90. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  91. package/dist/distribution/coordinator/in-memory.mjs +1 -2
  92. package/dist/distribution/transport/grpc.mjs +7 -2
  93. package/dist/distribution/transport/tcp.mjs +7 -1
  94. package/dist/distribution.d.ts +7 -7
  95. package/dist/distribution.mjs +1199 -1092
  96. package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
  97. package/dist/embed-4H7RTBVL.mjs +221 -0
  98. package/dist/embeddings/openai.mjs +2 -2
  99. package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
  100. package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
  101. package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
  102. package/dist/index.browser.mjs +4 -16
  103. package/dist/index.d.ts +1495 -78
  104. package/dist/index.mjs +14 -50
  105. package/dist/index.react-native.mjs +11 -28
  106. package/dist/invalidation/filesystem.mjs +1 -2
  107. package/dist/languages/albanian.mjs +1 -1
  108. package/dist/languages/amharic.mjs +1 -1
  109. package/dist/languages/arabic.mjs +1 -1
  110. package/dist/languages/armenian.mjs +1 -1
  111. package/dist/languages/azerbaijani.mjs +1 -1
  112. package/dist/languages/bambara.mjs +1 -1
  113. package/dist/languages/basque.mjs +1 -1
  114. package/dist/languages/belarusian.mjs +1 -1
  115. package/dist/languages/bengali.mjs +1 -1
  116. package/dist/languages/bosnian.mjs +1 -1
  117. package/dist/languages/breton.mjs +1 -1
  118. package/dist/languages/bulgarian.mjs +1 -1
  119. package/dist/languages/burmese.mjs +1 -1
  120. package/dist/languages/catalan.mjs +1 -1
  121. package/dist/languages/chinese.mjs +1 -1
  122. package/dist/languages/croatian.mjs +1 -1
  123. package/dist/languages/czech.mjs +1 -1
  124. package/dist/languages/dagbani.mjs +1 -1
  125. package/dist/languages/danish.mjs +1 -1
  126. package/dist/languages/dutch.mjs +1 -1
  127. package/dist/languages/english.mjs +1 -1
  128. package/dist/languages/esperanto.mjs +1 -1
  129. package/dist/languages/estonian.mjs +1 -1
  130. package/dist/languages/ewe.mjs +1 -1
  131. package/dist/languages/faroese.mjs +1 -1
  132. package/dist/languages/fijian.mjs +1 -1
  133. package/dist/languages/finnish.mjs +1 -1
  134. package/dist/languages/french.mjs +1 -1
  135. package/dist/languages/ga.mjs +1 -1
  136. package/dist/languages/galician.mjs +1 -1
  137. package/dist/languages/georgian.mjs +1 -1
  138. package/dist/languages/german.mjs +1 -1
  139. package/dist/languages/greek.mjs +1 -1
  140. package/dist/languages/guarani.mjs +1 -1
  141. package/dist/languages/gujarati.mjs +1 -1
  142. package/dist/languages/haitian-creole.mjs +1 -1
  143. package/dist/languages/hausa.mjs +1 -1
  144. package/dist/languages/hawaiian.mjs +1 -1
  145. package/dist/languages/hebrew.mjs +1 -1
  146. package/dist/languages/hindi.mjs +1 -1
  147. package/dist/languages/hungarian.mjs +1 -1
  148. package/dist/languages/icelandic.mjs +1 -1
  149. package/dist/languages/igbo.mjs +1 -1
  150. package/dist/languages/indonesian.mjs +1 -1
  151. package/dist/languages/irish.mjs +1 -1
  152. package/dist/languages/italian.mjs +1 -1
  153. package/dist/languages/japanese.mjs +1 -1
  154. package/dist/languages/kannada.mjs +1 -1
  155. package/dist/languages/kazakh.mjs +1 -1
  156. package/dist/languages/khmer.mjs +1 -1
  157. package/dist/languages/kinyarwanda.mjs +1 -1
  158. package/dist/languages/kirundi.mjs +1 -1
  159. package/dist/languages/korean.mjs +1 -1
  160. package/dist/languages/kurmanji.mjs +1 -1
  161. package/dist/languages/kyrgyz.mjs +1 -1
  162. package/dist/languages/lao.mjs +1 -1
  163. package/dist/languages/latin.mjs +1 -1
  164. package/dist/languages/latvian.mjs +1 -1
  165. package/dist/languages/lingala.mjs +1 -1
  166. package/dist/languages/lithuanian.mjs +1 -1
  167. package/dist/languages/luxembourgish.mjs +1 -1
  168. package/dist/languages/macedonian.mjs +1 -1
  169. package/dist/languages/malagasy.mjs +1 -1
  170. package/dist/languages/malay.mjs +1 -1
  171. package/dist/languages/malayalam.mjs +1 -1
  172. package/dist/languages/maltese.mjs +1 -1
  173. package/dist/languages/maori.mjs +1 -1
  174. package/dist/languages/marathi.mjs +1 -1
  175. package/dist/languages/nepali.mjs +1 -1
  176. package/dist/languages/norwegian.mjs +1 -1
  177. package/dist/languages/oromo.mjs +1 -1
  178. package/dist/languages/persian.mjs +1 -1
  179. package/dist/languages/polish.mjs +1 -1
  180. package/dist/languages/portuguese.mjs +1 -1
  181. package/dist/languages/punjabi.mjs +1 -1
  182. package/dist/languages/romanian.mjs +1 -1
  183. package/dist/languages/russian.mjs +1 -1
  184. package/dist/languages/samoan.mjs +1 -1
  185. package/dist/languages/sanskrit.mjs +1 -1
  186. package/dist/languages/scottish-gaelic.mjs +1 -1
  187. package/dist/languages/serbian.mjs +1 -1
  188. package/dist/languages/shona.mjs +1 -1
  189. package/dist/languages/sinhala.mjs +1 -1
  190. package/dist/languages/slovak.mjs +1 -1
  191. package/dist/languages/slovenian.mjs +1 -1
  192. package/dist/languages/sorani.mjs +1 -1
  193. package/dist/languages/spanish.mjs +1 -1
  194. package/dist/languages/swahili.mjs +1 -1
  195. package/dist/languages/swedish.mjs +1 -1
  196. package/dist/languages/tagalog.mjs +1 -1
  197. package/dist/languages/tamil.mjs +1 -1
  198. package/dist/languages/tatar.mjs +1 -1
  199. package/dist/languages/telugu.mjs +1 -1
  200. package/dist/languages/thai.mjs +1 -1
  201. package/dist/languages/tibetan.mjs +1 -1
  202. package/dist/languages/tigrinya.mjs +1 -1
  203. package/dist/languages/tongan.mjs +1 -1
  204. package/dist/languages/turkish.mjs +1 -1
  205. package/dist/languages/twi.mjs +1 -1
  206. package/dist/languages/ukrainian.mjs +1 -1
  207. package/dist/languages/urdu.mjs +1 -1
  208. package/dist/languages/vietnamese.mjs +1 -1
  209. package/dist/languages/welsh.mjs +1 -1
  210. package/dist/languages/wolof.mjs +1 -1
  211. package/dist/languages/xhosa.mjs +1 -1
  212. package/dist/languages/yoruba.mjs +1 -1
  213. package/dist/languages/zulu.mjs +1 -1
  214. package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
  215. package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
  216. package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
  217. package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
  218. package/dist/optional-writes-DDFHYUOH.mjs +218 -0
  219. package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
  220. package/dist/partial-words.browser.mjs +1649 -0
  221. package/dist/partial-words.mjs +15 -14
  222. package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
  223. package/dist/pattern.mjs +6 -8
  224. package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
  225. package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
  226. package/dist/phrase-module-XK3T44VU.mjs +634 -0
  227. package/dist/query-actions-OMXJEC2J.mjs +118 -0
  228. package/dist/query-suggestions/merge-worker.mjs +18 -54
  229. package/dist/query-suggestions.browser.mjs +26 -227
  230. package/dist/query-suggestions.d.ts +2 -2
  231. package/dist/query-suggestions.mjs +16 -67
  232. package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
  233. package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
  234. package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
  235. package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
  236. package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
  237. package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
  238. package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
  239. package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
  240. package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
  241. package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
  242. package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
  243. package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
  244. package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
  245. package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
  246. package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
  247. package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
  248. package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
  249. package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
  250. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  251. package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
  252. package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
  253. package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
  254. package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
  255. package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
  256. package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
  257. package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
  258. package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
  259. package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
  260. package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
  261. package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
  262. package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
  263. package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
  264. package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
  265. package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
  266. package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
  267. package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
  268. package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
  269. package/dist/react-native/partial-words.mjs +954 -35
  270. package/dist/react-native/pattern.mjs +238 -111
  271. package/dist/react-native/query-suggestions.mjs +33 -239
  272. package/dist/react-native/sync.mjs +95 -52
  273. package/dist/react-native/synonyms.mjs +256 -112
  274. package/dist/react.d.ts +9 -9
  275. package/dist/react.mjs +6 -0
  276. package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
  277. package/dist/rerankers/http.mjs +2 -2
  278. package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
  279. package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
  280. package/dist/runtime-4GKUIAKV.mjs +595 -0
  281. package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
  282. package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
  283. package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
  284. package/dist/server.d.ts +33 -13
  285. package/dist/server.mjs +81 -100
  286. package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
  287. package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
  288. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  289. package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
  290. package/dist/sync/filesystem.d.ts +3 -3
  291. package/dist/sync/filesystem.mjs +27 -11
  292. package/dist/sync/indexeddb.browser.mjs +287 -0
  293. package/dist/sync/indexeddb.d.ts +3 -3
  294. package/dist/sync/indexeddb.mjs +7 -2
  295. package/dist/sync.browser.mjs +88 -72
  296. package/dist/sync.d.ts +4 -4
  297. package/dist/sync.mjs +89 -59
  298. package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
  299. package/dist/synonyms.mjs +15 -11
  300. package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
  301. package/dist/thread-SNILJXX3.mjs +30 -0
  302. package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
  303. package/dist/vector/search-worker.d.ts +7 -1
  304. package/dist/vector/search-worker.mjs +9 -13
  305. package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
  306. package/dist/workers/entry.mjs +60 -30
  307. package/openapi.json +66 -50
  308. package/package.json +233 -8
  309. package/dist/analysis-change-CKEMM2MC.mjs +0 -55
  310. package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
  311. package/dist/archive-history-E4R73RWD.mjs +0 -29
  312. package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
  313. package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
  314. package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
  315. package/dist/browser/chunk-25MTVDYD.mjs +0 -432
  316. package/dist/browser/chunk-3I27KXQD.mjs +0 -34
  317. package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
  318. package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
  319. package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
  320. package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
  321. package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
  322. package/dist/browser/chunk-AI5UI46D.mjs +0 -12
  323. package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
  324. package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
  325. package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
  326. package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
  327. package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
  328. package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
  329. package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
  330. package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
  331. package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
  332. package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
  333. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  334. package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
  335. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  336. package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
  337. package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
  338. package/dist/browser/chunk-MY2XKH22.mjs +0 -40
  339. package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
  340. package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
  341. package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
  342. package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
  343. package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
  344. package/dist/browser/chunk-TK63WFXK.mjs +0 -90
  345. package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
  346. package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
  347. package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
  348. package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
  349. package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
  350. package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
  351. package/dist/browser/chunk-V43N6TZO.mjs +0 -857
  352. package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
  353. package/dist/browser/chunk-WN464PTL.mjs +0 -375
  354. package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
  355. package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
  356. package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
  357. package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
  358. package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
  359. package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
  360. package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
  361. package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
  362. package/dist/browser/import-5ESVMFDJ.mjs +0 -20
  363. package/dist/browser/methods-KTQCSIN4.mjs +0 -30
  364. package/dist/browser/module-NDSQLPMQ.mjs +0 -25
  365. package/dist/browser/operations-OJ3BXS55.mjs +0 -5
  366. package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
  367. package/dist/browser/pool-3JLET5RO.mjs +0 -297
  368. package/dist/browser/release-HYXPVJVS.mjs +0 -32
  369. package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
  370. package/dist/browser/retention-TY3L3BFN.mjs +0 -211
  371. package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
  372. package/dist/browser/start-2UZTOBG5.mjs +0 -26
  373. package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
  374. package/dist/browser/write-ULXAN5B4.mjs +0 -196
  375. package/dist/chunk-2NE7VCYE.mjs +0 -212
  376. package/dist/chunk-2ROUFMQX.mjs +0 -543
  377. package/dist/chunk-37CD3OKS.mjs +0 -87
  378. package/dist/chunk-3MXV3VHR.mjs +0 -46
  379. package/dist/chunk-3OH67G7R.mjs +0 -214
  380. package/dist/chunk-4B7JIGQD.mjs +0 -106
  381. package/dist/chunk-5YNRKI2C.mjs +0 -81
  382. package/dist/chunk-6HTGOXIK.mjs +0 -875
  383. package/dist/chunk-6OEGXLBX.mjs +0 -57
  384. package/dist/chunk-6RC4K46J.mjs +0 -86
  385. package/dist/chunk-7GFE3PEY.mjs +0 -40
  386. package/dist/chunk-7GXJUJ24.mjs +0 -49
  387. package/dist/chunk-7V5IUP2T.mjs +0 -63
  388. package/dist/chunk-A4L7YTKX.mjs +0 -458
  389. package/dist/chunk-BEZH7VEH.mjs +0 -23
  390. package/dist/chunk-BNL5OPRV.mjs +0 -634
  391. package/dist/chunk-BTPGQC6Z.mjs +0 -27
  392. package/dist/chunk-C7XQYG6U.mjs +0 -50
  393. package/dist/chunk-CQZXZ75V.mjs +0 -562
  394. package/dist/chunk-CWS55TEX.mjs +0 -242
  395. package/dist/chunk-CZQ33DLW.mjs +0 -78
  396. package/dist/chunk-EEBCDP36.mjs +0 -32
  397. package/dist/chunk-FB6463IN.mjs +0 -1499
  398. package/dist/chunk-FCJ553HS.mjs +0 -655
  399. package/dist/chunk-FEZVDOWX.mjs +0 -6996
  400. package/dist/chunk-FOCV4CBZ.mjs +0 -87
  401. package/dist/chunk-FQBFZAWG.mjs +0 -27
  402. package/dist/chunk-G6HRSGOU.mjs +0 -74
  403. package/dist/chunk-GES7U2PM.mjs +0 -68
  404. package/dist/chunk-GH4WQ22V.mjs +0 -34
  405. package/dist/chunk-GPWMUPCA.mjs +0 -1126
  406. package/dist/chunk-GQPTDQCL.mjs +0 -54
  407. package/dist/chunk-HJSN7UYG.mjs +0 -10
  408. package/dist/chunk-I54T6CQP.mjs +0 -117
  409. package/dist/chunk-IPN7S4KX.mjs +0 -130
  410. package/dist/chunk-J4IJTTEX.mjs +0 -226
  411. package/dist/chunk-JSV6OQ4C.mjs +0 -74
  412. package/dist/chunk-K4ZIGL2O.mjs +0 -290
  413. package/dist/chunk-KAKQ74LS.mjs +0 -53
  414. package/dist/chunk-KCQYFO4K.mjs +0 -62
  415. package/dist/chunk-KHKIRS5D.mjs +0 -113
  416. package/dist/chunk-KOKM4PQN.mjs +0 -52
  417. package/dist/chunk-KP7TYHWT.mjs +0 -92
  418. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  419. package/dist/chunk-L5ORED7V.mjs +0 -25
  420. package/dist/chunk-LDNCLRZU.mjs +0 -218
  421. package/dist/chunk-LHKOSI2J.mjs +0 -58
  422. package/dist/chunk-LQG5G2LY.mjs +0 -39
  423. package/dist/chunk-MATMVH65.mjs +0 -9
  424. package/dist/chunk-MP7ABPTD.mjs +0 -189
  425. package/dist/chunk-MW6BR4QP.mjs +0 -316
  426. package/dist/chunk-MWD5VPN7.mjs +0 -21
  427. package/dist/chunk-ODYKO5TO.mjs +0 -101
  428. package/dist/chunk-OG7UGJ6F.mjs +0 -63
  429. package/dist/chunk-ORN5BLL7.mjs +0 -11
  430. package/dist/chunk-OYABFKTW.mjs +0 -1974
  431. package/dist/chunk-P5LANSH6.mjs +0 -121
  432. package/dist/chunk-PELHVTO7.mjs +0 -24
  433. package/dist/chunk-PIH5SJ4N.mjs +0 -7481
  434. package/dist/chunk-PSZI4O24.mjs +0 -253
  435. package/dist/chunk-PV2CG7EJ.mjs +0 -886
  436. package/dist/chunk-PWGYQXML.mjs +0 -470
  437. package/dist/chunk-Q2UL2TPC.mjs +0 -21
  438. package/dist/chunk-QHPYS427.mjs +0 -66
  439. package/dist/chunk-QJVOLKU6.mjs +0 -111
  440. package/dist/chunk-QLDABJZM.mjs +0 -20
  441. package/dist/chunk-QYRNLLLI.mjs +0 -99
  442. package/dist/chunk-RCX4WNLC.mjs +0 -151
  443. package/dist/chunk-RFBLQLLZ.mjs +0 -16
  444. package/dist/chunk-RIHIVFIQ.mjs +0 -240
  445. package/dist/chunk-RLOES5SX.mjs +0 -31
  446. package/dist/chunk-RWURUDSP.mjs +0 -2639
  447. package/dist/chunk-RZWWB2XR.mjs +0 -96
  448. package/dist/chunk-S5LN746F.mjs +0 -10
  449. package/dist/chunk-SOI6LTHM.mjs +0 -216
  450. package/dist/chunk-SQJWAIF4.mjs +0 -12
  451. package/dist/chunk-TDIK6NTV.mjs +0 -10
  452. package/dist/chunk-TQXHPFWT.mjs +0 -177
  453. package/dist/chunk-U55XJEIG.mjs +0 -18
  454. package/dist/chunk-UD5HIS4U.mjs +0 -382
  455. package/dist/chunk-UHICS3JI.mjs +0 -83
  456. package/dist/chunk-UJDJB3NE.mjs +0 -913
  457. package/dist/chunk-VD7J552I.mjs +0 -28
  458. package/dist/chunk-VSLSGASN.mjs +0 -30
  459. package/dist/chunk-VZ2ONHEE.mjs +0 -3715
  460. package/dist/chunk-W7OSSKH5.mjs +0 -1427
  461. package/dist/chunk-WZ7YV54N.mjs +0 -152
  462. package/dist/chunk-X5CV2E2B.mjs +0 -1
  463. package/dist/chunk-Y4WPNHMC.mjs +0 -50
  464. package/dist/chunk-YEKL4JBK.mjs +0 -96
  465. package/dist/chunk-ZXNQEW43.mjs +0 -18
  466. package/dist/chunk-ZYAY26KG.mjs +0 -29
  467. package/dist/direct-executor-V5DRIECI.mjs +0 -34
  468. package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
  469. package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
  470. package/dist/gzip-F4HMIEEQ.mjs +0 -1
  471. package/dist/import-GPKLTSKA.mjs +0 -56
  472. package/dist/methods-7IOINERF.mjs +0 -30
  473. package/dist/module-JYHFTVPS.mjs +0 -63
  474. package/dist/operations-G5EP6JPX.mjs +0 -5
  475. package/dist/phrase-module-XW5MKWP5.mjs +0 -330
  476. package/dist/pool-Y4A46T36.mjs +0 -298
  477. package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
  478. package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
  479. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
  480. package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
  481. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
  482. package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
  483. package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
  484. package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
  485. package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
  486. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
  487. package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
  488. package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
  489. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
  490. package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
  491. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
  492. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
  493. package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
  494. package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
  495. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
  496. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
  497. package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
  498. package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
  499. package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
  500. package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
  501. package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
  502. package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
  503. package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
  504. package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
  505. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
  506. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
  507. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
  508. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
  509. package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
  510. package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
  511. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
  512. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
  513. package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
  514. package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
  515. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
  516. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
  517. package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
  518. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
  519. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
  520. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
  521. package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
  522. package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
  523. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
  524. package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
  525. package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
  526. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
  527. package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
  528. package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
  529. package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
  530. package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
  531. package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
  532. package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
  533. package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
  534. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
  535. package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
  536. package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
  537. package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
  538. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
  539. package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
  540. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
  541. package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
  542. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
  543. package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
  544. package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
  545. package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
  546. package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
  547. package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
  548. package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
  549. package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
  550. package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
  551. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
  552. package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
  553. package/dist/release-SMR4WJD4.mjs +0 -32
  554. package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
  555. package/dist/restore-queue-LZH7LMLR.mjs +0 -55
  556. package/dist/retention-3K7ARGAG.mjs +0 -66
  557. package/dist/runtime-COIUORCN.mjs +0 -177
  558. package/dist/snapshot-O7ICP7NJ.mjs +0 -783
  559. package/dist/start-S4PFP6RS.mjs +0 -66
  560. package/dist/sync-state-ZWHLAFLE.mjs +0 -15
  561. package/dist/thread-YLJ3ZPQ3.mjs +0 -50
  562. package/dist/write-2OBJGLFS.mjs +0 -233
  563. /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
  564. /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
  565. /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
@@ -1 +0,0 @@
1
- export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-IAENKE2D.mjs';
@@ -1,29 +0,0 @@
1
- import { historyFilesOf, listHistoryFolder, historyFileOf } from './chunk-ZZT7CEN7.mjs';
2
- import { attachedRuntimeOf } from './chunk-DIYDOIM4.mjs';
3
- import { QUERY_HISTORY_ARCHIVE_PREFIX } from './chunk-PJHAB5ZY.mjs';
4
-
5
- // src/query-suggestions/archive-history.ts
6
- function inDayOrder(left, right) {
7
- if (left.day !== right.day) return left.day < right.day ? -1 : 1;
8
- return left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
9
- }
10
- async function* historyArchiveEntries(core, indexName) {
11
- const runtime = attachedRuntimeOf(core);
12
- const files = runtime?.host.files ?? historyFilesOf(core.durability?.manager.historyStorage?.() ?? null);
13
- await runtime?.saveDay(indexName);
14
- const names = [];
15
- for (const key of await listHistoryFolder(files, indexName)) {
16
- const name = historyFileOf(indexName, key);
17
- if (name !== null) names.push(name);
18
- }
19
- names.sort(inDayOrder);
20
- let position = 0;
21
- for (const name of names) {
22
- const bytes = await files.read(name.key);
23
- if (bytes === null || bytes === "too-long") continue;
24
- yield { name: `${QUERY_HISTORY_ARCHIVE_PREFIX}${position}`, bytes };
25
- position += 1;
26
- }
27
- }
28
-
29
- export { historyArchiveEntries };
@@ -1,529 +0,0 @@
1
- import { withUnpairedSurrogatesReplaced, compareCodePoints } from './chunk-EJ33LCHV.mjs';
2
- import { LOOKUP_KEY_SCRATCH_BYTES } from './chunk-FTZHMRD2.mjs';
3
- import { createNarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
4
-
5
- // src/core/tokenizer/constants.ts
6
- var TOKEN_CACHE_SIZE_FLOOR = 5e4;
7
- var TOKEN_CACHE_SIZE_CEILING = 2e6;
8
- var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
9
- var NODE_TOKEN_CACHE_ENTRIES = 1e6;
10
- var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
11
- var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
12
- var SMALL_DEVICE_MEMORY_GB = 1;
13
- var MEDIUM_DEVICE_MEMORY_GB = 4;
14
- var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
15
- var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
16
- var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
17
- var NATIVE_ANALYSIS_FIRST_TOKEN_RECORDS = 256;
18
- var NATIVE_ANALYSIS_HELD_TERMS = 16384;
19
- var DEFAULT_MIN_TOKEN_LENGTH = 1;
20
- var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
21
-
22
- // src/core/tokenizer/native-hook.ts
23
- var installed = null;
24
- function installNativeAnalysis(analysis) {
25
- installed = analysis;
26
- }
27
- function nativeAnalysis() {
28
- return installed;
29
- }
30
-
31
- // src/core/tokenizer/normalize.ts
32
- var DOTTED_CAPITAL_I = "\u0130";
33
- var COMBINING_DOT_ABOVE = "\u0307";
34
- var ARMENIAN_MARKS = /[\u055B-\u055F]/;
35
- var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
36
- var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
37
- var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
38
- var FULLWIDTH_ASCII_END = 65374;
39
- var FULLWIDTH_ASCII_OFFSET = 65248;
40
- var HALFWIDTH_KANA_START = 65377;
41
- var HALFWIDTH_VOICED_MARK = 65438;
42
- var HALFWIDTH_SEMI_VOICED_MARK = 65439;
43
- var COMBINING_VOICED_MARK = "\u3099";
44
- var COMBINING_SEMI_VOICED_MARK = "\u309A";
45
- var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
46
- function foldForm(match) {
47
- if (match === DOTTED_CAPITAL_I) return "i";
48
- if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
49
- return "'";
50
- }
51
- function foldWidth(match) {
52
- const codePoint = match.charCodeAt(0);
53
- if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
54
- if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
55
- if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
56
- return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
57
- }
58
- function isAscii(text) {
59
- for (let i = 0; i < text.length; i++) {
60
- if (text.charCodeAt(i) > 127) return false;
61
- }
62
- return true;
63
- }
64
- function foldWidthForms(text) {
65
- if (!WIDTH_FORMS.test(text)) return text;
66
- return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
67
- }
68
- function normalizeForSplitting(text) {
69
- if (isAscii(text)) return text.toLowerCase();
70
- return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
71
- }
72
- function stripDiacritics(text) {
73
- return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
74
- }
75
-
76
- // src/core/tokenizer/cache.ts
77
- function clampCacheSize(entries) {
78
- return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
79
- }
80
- function computeDefaultCacheSize() {
81
- try {
82
- if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
83
- const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
84
- if (constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER) {
85
- const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
86
- return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
87
- }
88
- return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
89
- }
90
- if (typeof navigator !== "undefined") {
91
- const mem = navigator.deviceMemory;
92
- if (typeof mem === "number" && mem > 0) {
93
- let entries;
94
- if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
95
- else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
96
- else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
97
- return clampCacheSize(entries);
98
- }
99
- }
100
- } catch {
101
- return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
102
- }
103
- return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
104
- }
105
- var normalizationCache = /* @__PURE__ */ new Map();
106
- var normalizationCacheSize = 0;
107
- var maxCacheSize = computeDefaultCacheSize();
108
- var cachedLangName = "";
109
- var cachedFlags = "";
110
- var cachedBucket = null;
111
- function configureNormalizationCache(maxSize) {
112
- if (!Number.isFinite(maxSize)) {
113
- throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
114
- received: maxSize
115
- });
116
- }
117
- if (maxSize < 0) {
118
- throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
119
- received: maxSize
120
- });
121
- }
122
- if (maxSize === 0) {
123
- throw createNarsilError(
124
- ErrorCodes.CONFIG_INVALID,
125
- "tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
126
- {
127
- received: maxSize
128
- }
129
- );
130
- }
131
- maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
132
- if (normalizationCacheSize > maxCacheSize) {
133
- evictOldestEntries(normalizationCacheSize - maxCacheSize);
134
- }
135
- }
136
- function getCacheBucket(language, flags) {
137
- const langKey = `${language.name}@${language.revision}`;
138
- if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
139
- const key = `${langKey}:${flags}`;
140
- let bucket = normalizationCache.get(key);
141
- if (!bucket) {
142
- bucket = /* @__PURE__ */ new Map();
143
- normalizationCache.set(key, bucket);
144
- }
145
- cachedLangName = langKey;
146
- cachedFlags = flags;
147
- cachedBucket = bucket;
148
- return bucket;
149
- }
150
- function evictOldestEntries(count) {
151
- let remaining = count;
152
- for (const [key, bucket] of normalizationCache) {
153
- if (remaining <= 0) break;
154
- if (bucket.size <= remaining) {
155
- remaining -= bucket.size;
156
- normalizationCacheSize -= bucket.size;
157
- normalizationCache.delete(key);
158
- if (bucket === cachedBucket) cachedBucket = null;
159
- } else {
160
- let deleted = 0;
161
- for (const raw of bucket.keys()) {
162
- if (deleted >= remaining) break;
163
- bucket.delete(raw);
164
- deleted++;
165
- }
166
- normalizationCacheSize -= deleted;
167
- remaining = 0;
168
- }
169
- }
170
- }
171
- function transformInTypeScript(raw, language, stem, removeDiacritics) {
172
- let normalized = raw;
173
- if (language.normalizer) normalized = language.normalizer(normalized);
174
- if (removeDiacritics) normalized = stripDiacritics(normalized);
175
- if (stem && language.stemmer) normalized = language.stemmer(normalized);
176
- return normalized;
177
- }
178
- function transformToken(raw, language, stem, removeDiacritics) {
179
- const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
180
- let bucket = getCacheBucket(language, flags);
181
- const cached = bucket.get(raw);
182
- if (cached !== void 0) return cached;
183
- const normalized = withUnpairedSurrogatesReplaced(
184
- nativeAnalysis()?.transform(raw, language, stem, removeDiacritics) ?? transformInTypeScript(raw, language, stem, removeDiacritics)
185
- );
186
- if (normalizationCacheSize >= maxCacheSize) {
187
- evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
188
- bucket = getCacheBucket(language, flags);
189
- }
190
- bucket.set(raw, normalized);
191
- normalizationCacheSize++;
192
- return normalized;
193
- }
194
- function clearNormalizationCache() {
195
- normalizationCache.clear();
196
- normalizationCacheSize = 0;
197
- cachedBucket = null;
198
- }
199
- function resetNormalizationCache() {
200
- clearNormalizationCache();
201
- maxCacheSize = computeDefaultCacheSize();
202
- }
203
- function getNormalizationCacheSize() {
204
- return normalizationCacheSize;
205
- }
206
-
207
- // src/core/tokenizer/ngrams.ts
208
- var OTHER_SCRIPT = 0;
209
- var HAN_SCRIPT = 1;
210
- var HIRAGANA_SCRIPT = 2;
211
- var KATAKANA_SCRIPT = 3;
212
- var HANGUL_SCRIPT = 4;
213
- var THAI_SCRIPT = 5;
214
- var LAO_SCRIPT = 6;
215
- var KHMER_SCRIPT = 7;
216
- var MYANMAR_SCRIPT = 8;
217
- var PROLONGED_SOUND_MARK = 12540;
218
- var IDEOGRAPHIC_ITERATION_MARK = 12293;
219
- var LOWEST_NGRAM_CODE_POINT = 3584;
220
- var LOWEST_MARK_CODE_POINT = 768;
221
- var THAI_BLOCK_END = 3711;
222
- var LAO_BLOCK_END = 3839;
223
- var MYANMAR_BLOCK_START = 4096;
224
- var MYANMAR_BLOCK_END = 4255;
225
- var KHMER_BLOCK_START = 6016;
226
- var KHMER_BLOCK_END = 6655;
227
- var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
228
- var HIRAGANA_BLOCK_START = 12352;
229
- var HIRAGANA_BLOCK_END = 12447;
230
- var COMBINING_VOICED_MARK2 = 12441;
231
- var COMBINING_SEMI_VOICED_MARK2 = 12442;
232
- var KATAKANA_BLOCK_START = 12448;
233
- var KATAKANA_BLOCK_END = 12543;
234
- var HAN_EXTENSION_A_START = 13312;
235
- var HAN_EXTENSION_A_END = 19903;
236
- var HAN_UNIFIED_START = 19968;
237
- var HAN_UNIFIED_END = 40959;
238
- var HANGUL_SYLLABLE_START = 44032;
239
- var HANGUL_SYLLABLE_END = 55203;
240
- var HAN_COMPATIBILITY_START = 63744;
241
- var HAN_COMPATIBILITY_END = 64255;
242
- var SUPPLEMENTARY_PLANE_START = 65536;
243
- var MARK = /\p{M}/uy;
244
- var HAN = /\p{Script=Han}/uy;
245
- var HIRAGANA = /\p{Script=Hiragana}/uy;
246
- var KATAKANA = /\p{Script=Katakana}/uy;
247
- var HANGUL = /\p{Script=Hangul}/uy;
248
- var THAI = /\p{Script=Thai}/uy;
249
- var LAO = /\p{Script=Lao}/uy;
250
- var KHMER = /\p{Script=Khmer}/uy;
251
- var MYANMAR = /\p{Script=Myanmar}/uy;
252
- function matchesAt(pattern, part, index) {
253
- pattern.lastIndex = index;
254
- return pattern.test(part);
255
- }
256
- function isMark(part, index, codePoint) {
257
- if (codePoint < LOWEST_MARK_CODE_POINT) return false;
258
- if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
259
- if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
260
- return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
261
- }
262
- if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
263
- if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
264
- return matchesAt(MARK, part, index);
265
- }
266
- function classifyOutsideCommonBlocks(part, index) {
267
- if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
268
- if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
269
- if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
270
- if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
271
- if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
272
- if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
273
- if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
274
- if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
275
- return OTHER_SCRIPT;
276
- }
277
- function classifySoutheastAsian(part, index, codePoint) {
278
- if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
279
- if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
280
- if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
281
- return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
282
- }
283
- if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
284
- return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
285
- }
286
- return classifyOutsideCommonBlocks(part, index);
287
- }
288
- function classify(part, index, codePoint, current) {
289
- if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
290
- if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
291
- if (codePoint === PROLONGED_SOUND_MARK) {
292
- return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
293
- }
294
- if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
295
- if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
296
- if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
297
- if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
298
- if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
299
- if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
300
- if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
301
- return classifyOutsideCommonBlocks(part, index);
302
- }
303
- function emitRun(part, offsets, kind, size, out) {
304
- const count = offsets.length - 1;
305
- if (count < 1) return;
306
- if (kind === OTHER_SCRIPT || count <= size) {
307
- out.push(part.slice(offsets[0], offsets[count]));
308
- return;
309
- }
310
- for (let start = 0; start + size <= count; start++) {
311
- out.push(part.slice(offsets[start], offsets[start + size]));
312
- }
313
- }
314
- function expandNgrams(part, size, out) {
315
- const length = part.length;
316
- if (length === 0) return;
317
- const offsets = [];
318
- let kind = OTHER_SCRIPT;
319
- let index = 0;
320
- while (index < length) {
321
- const codePoint = part.codePointAt(index);
322
- if (codePoint === void 0) break;
323
- const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
324
- if (offsets.length > 0 && isMark(part, index, codePoint)) {
325
- index += width;
326
- continue;
327
- }
328
- const next = classify(part, index, codePoint, kind);
329
- if (offsets.length > 0 && next !== kind) {
330
- offsets.push(index);
331
- emitRun(part, offsets, kind, size, out);
332
- offsets.length = 0;
333
- }
334
- kind = next;
335
- offsets.push(index);
336
- index += width;
337
- }
338
- offsets.push(length);
339
- emitRun(part, offsets, kind, size, out);
340
- }
341
-
342
- // src/core/tokenizer/tokenize.ts
343
- function producesSurfaceForms(language, options) {
344
- if (!options?.collectSurfaces || options.customTokenizer) return false;
345
- return (options.stem ?? true) && language.stemmer !== null;
346
- }
347
- function resolveStopWords(language, override) {
348
- if (!override) return language.stopWords;
349
- if (override instanceof Set) return override;
350
- return override(language.stopWords);
351
- }
352
- function splitText(text, language) {
353
- const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
354
- const parts = text.split(pattern);
355
- const ngramSize = language.tokenizer?.ngramSize;
356
- if (ngramSize === void 0 || ngramSize < 1) return parts;
357
- const size = Math.floor(ngramSize);
358
- const expanded = [];
359
- for (const part of parts) {
360
- expandNgrams(part, size, expanded);
361
- }
362
- return expanded;
363
- }
364
- var CHAR_APOSTROPHE = 39;
365
- var CHAR_S = 115;
366
- function stripPossessive(token) {
367
- const len = token.length;
368
- if (len >= 2 && token.charCodeAt(len - 1) === CHAR_S && token.charCodeAt(len - 2) === CHAR_APOSTROPHE) {
369
- return token.slice(0, -2);
370
- }
371
- if (len >= 1 && token.charCodeAt(len - 1) === CHAR_APOSTROPHE) {
372
- return token.slice(0, -1);
373
- }
374
- return token;
375
- }
376
- function customTokensOf(tokenizer, text) {
377
- const entries = tokenizer.tokenize(text);
378
- let repaired = null;
379
- for (let at = 0; at < entries.length; at++) {
380
- const entry = entries[at];
381
- const token = withUnpairedSurrogatesReplaced(entry.token);
382
- if (token === entry.token) continue;
383
- repaired ??= entries.slice();
384
- repaired[at] = { token, position: entry.position };
385
- }
386
- return repaired ?? entries;
387
- }
388
- function visitTokens(text, language, options, visit) {
389
- const {
390
- stem = true,
391
- removeStopWords = true,
392
- removeDiacritics = false,
393
- collectSurfaces = false,
394
- customTokenizer,
395
- stopWordOverride
396
- } = options ?? {};
397
- if (customTokenizer) {
398
- for (const entry of customTokensOf(customTokenizer, text)) {
399
- visit(entry.token, entry.position, void 0, entry.token);
400
- }
401
- return;
402
- }
403
- if (nativeAnalysis()?.visitTokens(text, language, options, visit) === true) return;
404
- const rawParts = splitText(normalizeForSplitting(text), language);
405
- const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
406
- const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
407
- const wantSurfaces = collectSurfaces && stem && language.stemmer !== null;
408
- const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : null;
409
- const stripPossessives = language.tokenizer?.stripPossessive ?? false;
410
- for (let position = 0; position < rawParts.length; position++) {
411
- const part = rawParts[position];
412
- const candidate = stripPossessives ? stripPossessive(part) : part;
413
- if (candidate.length < minLength || stopWords?.has(candidate)) continue;
414
- const processed = transformToken(candidate, language, stem, effectiveDiacritics);
415
- if (processed.length === 0) continue;
416
- if (!wantSurfaces) {
417
- visit(processed, position, void 0, part);
418
- continue;
419
- }
420
- const shown = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
421
- visit(processed, position, surfaceOf(shown, processed), part);
422
- }
423
- }
424
- function surfaceOf(shown, term) {
425
- if (shown === term) return void 0;
426
- const surface = withUnpairedSurrogatesReplaced(shown);
427
- return surface === term ? void 0 : surface;
428
- }
429
- function visitSurfaceSpellings(text, language, visit) {
430
- if (nativeAnalysis()?.visitSpellings(text, language, visit) === true) return;
431
- const rawParts = splitText(normalizeForSplitting(text), language);
432
- const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
433
- const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
434
- const stripPossessives = language.tokenizer?.stripPossessive ?? false;
435
- for (const part of rawParts) {
436
- const candidate = stripPossessives ? stripPossessive(part) : part;
437
- if (candidate.length < minLength) continue;
438
- visit(
439
- removeDiacritics ? transformToken(candidate, language, false, true) : withUnpairedSurrogatesReplaced(candidate)
440
- );
441
- }
442
- }
443
- function tokenize(text, language, options) {
444
- if (options?.customTokenizer) {
445
- const customResult = customTokensOf(options.customTokenizer, text);
446
- return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
447
- }
448
- const tokens = [];
449
- const originalTokens = [];
450
- const surfaces = producesSurfaceForms(language, options) ? [] : void 0;
451
- visitTokens(text, language, options, (token, position, surface, original) => {
452
- tokens.push({ token, position });
453
- originalTokens.push(original);
454
- surfaces?.push(surface);
455
- });
456
- return { tokens, originalTokens, surfaces };
457
- }
458
-
459
- // src/core/partition/frozen/string-blob.ts
460
- var SURROGATE = /[\ud800-\udfff]/;
461
- var UTF8_BYTES_PER_UTF16_UNIT = 3;
462
- var encoder = new TextEncoder();
463
- var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
464
- function encodeLookupKey(text) {
465
- if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder.encode(text);
466
- const { written } = encoder.encodeInto(text, lookupKeyBytes);
467
- return lookupKeyBytes.subarray(0, written);
468
- }
469
- function encodeOneByOne(strings) {
470
- const encoded = new Array(strings.length);
471
- let blobLength = 0;
472
- for (let i = 0; i < strings.length; i++) {
473
- encoded[i] = encoder.encode(strings[i]);
474
- blobLength += encoded[i].length;
475
- }
476
- const blob = new Uint8Array(blobLength);
477
- const offsets = new Uint32Array(strings.length + 1);
478
- let cursor = 0;
479
- for (let i = 0; i < strings.length; i++) {
480
- blob.set(encoded[i], cursor);
481
- cursor += encoded[i].length;
482
- offsets[i + 1] = cursor;
483
- }
484
- return { blob, offsets };
485
- }
486
- function basicPlaneUtf8Bytes(text) {
487
- let bytes = text.length;
488
- for (let i = 0; i < text.length; i++) {
489
- const unit = text.charCodeAt(i);
490
- if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
491
- }
492
- return bytes;
493
- }
494
- function encodeStringBlob(strings) {
495
- const joined = strings.join("");
496
- if (SURROGATE.test(joined)) return encodeOneByOne(strings);
497
- const blob = encoder.encode(joined);
498
- const offsets = new Uint32Array(strings.length + 1);
499
- let cursor = 0;
500
- for (let i = 0; i < strings.length; i++) {
501
- cursor += basicPlaneUtf8Bytes(strings[i]);
502
- offsets[i + 1] = cursor;
503
- }
504
- return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
505
- }
506
- function orderByComparator(strings) {
507
- const order = new Array(strings.length);
508
- for (let i = 0; i < strings.length; i++) order[i] = i;
509
- return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
510
- }
511
- function alreadyInCodePointOrder(strings) {
512
- for (let i = 1; i < strings.length; i++) {
513
- if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
514
- }
515
- return true;
516
- }
517
- function codePointOrder(strings) {
518
- if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
519
- if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
520
- const positions = /* @__PURE__ */ new Map();
521
- for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
522
- if (positions.size !== strings.length) return orderByComparator(strings);
523
- const order = new Array(strings.length);
524
- const sorted = [...strings].sort();
525
- for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
526
- return order;
527
- }
528
-
529
- export { NATIVE_ANALYSIS_FIRST_TOKEN_RECORDS, NATIVE_ANALYSIS_HELD_TERMS, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, getNormalizationCacheSize, installNativeAnalysis, resetNormalizationCache, resolveStopWords, tokenize, transformToken, visitSurfaceSpellings, visitTokens };