opencode-rehydar 0.14.1 → 0.14.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (430) hide show
  1. package/dist/browser.d.ts +54 -0
  2. package/dist/browser.d.ts.map +1 -0
  3. package/dist/browser.js +74 -0
  4. package/dist/browser.js.map +1 -0
  5. package/dist/core/anonymizer.d.ts +309 -0
  6. package/dist/core/anonymizer.d.ts.map +1 -0
  7. package/dist/core/anonymizer.js +555 -0
  8. package/dist/core/anonymizer.js.map +1 -0
  9. package/dist/core/index.d.ts +6 -0
  10. package/dist/core/index.d.ts.map +1 -0
  11. package/dist/core/index.js +6 -0
  12. package/dist/core/index.js.map +1 -0
  13. package/dist/crypto/index.d.ts +6 -0
  14. package/dist/crypto/index.d.ts.map +1 -0
  15. package/dist/crypto/index.js +6 -0
  16. package/dist/crypto/index.js.map +1 -0
  17. package/dist/crypto/pii-map-crypto.d.ts +114 -0
  18. package/dist/crypto/pii-map-crypto.d.ts.map +1 -0
  19. package/dist/crypto/pii-map-crypto.js +228 -0
  20. package/dist/crypto/pii-map-crypto.js.map +1 -0
  21. package/dist/index.d.ts +53 -0
  22. package/dist/index.d.ts.map +1 -0
  23. package/dist/index.js +72 -0
  24. package/dist/index.js.map +1 -0
  25. package/dist/ner/bio-decoder.d.ts +64 -0
  26. package/dist/ner/bio-decoder.d.ts.map +1 -0
  27. package/dist/ner/bio-decoder.js +218 -0
  28. package/dist/ner/bio-decoder.js.map +1 -0
  29. package/dist/ner/index.d.ts +12 -0
  30. package/dist/ner/index.d.ts.map +1 -0
  31. package/dist/ner/index.js +12 -0
  32. package/dist/ner/index.js.map +1 -0
  33. package/dist/ner/inference-server-client.d.ts +81 -0
  34. package/dist/ner/inference-server-client.d.ts.map +1 -0
  35. package/dist/ner/inference-server-client.js +120 -0
  36. package/dist/ner/inference-server-client.js.map +1 -0
  37. package/dist/ner/inference-server-ner-model.d.ts +43 -0
  38. package/dist/ner/inference-server-ner-model.d.ts.map +1 -0
  39. package/dist/ner/inference-server-ner-model.js +119 -0
  40. package/dist/ner/inference-server-ner-model.js.map +1 -0
  41. package/dist/ner/model-manager.d.ts +111 -0
  42. package/dist/ner/model-manager.d.ts.map +1 -0
  43. package/dist/ner/model-manager.js +334 -0
  44. package/dist/ner/model-manager.js.map +1 -0
  45. package/dist/ner/ner-model.d.ts +172 -0
  46. package/dist/ner/ner-model.d.ts.map +1 -0
  47. package/dist/ner/ner-model.js +514 -0
  48. package/dist/ner/ner-model.js.map +1 -0
  49. package/dist/ner/onnx-runtime.browser.d.ts +46 -0
  50. package/dist/ner/onnx-runtime.browser.d.ts.map +1 -0
  51. package/dist/ner/onnx-runtime.browser.js +90 -0
  52. package/dist/ner/onnx-runtime.browser.js.map +1 -0
  53. package/dist/ner/onnx-runtime.d.ts +72 -0
  54. package/dist/ner/onnx-runtime.d.ts.map +1 -0
  55. package/dist/ner/onnx-runtime.js +132 -0
  56. package/dist/ner/onnx-runtime.js.map +1 -0
  57. package/dist/ner/token-window.d.ts +19 -0
  58. package/dist/ner/token-window.d.ts.map +1 -0
  59. package/dist/ner/token-window.js +101 -0
  60. package/dist/ner/token-window.js.map +1 -0
  61. package/dist/ner/tokenizer.d.ts +142 -0
  62. package/dist/ner/tokenizer.d.ts.map +1 -0
  63. package/dist/ner/tokenizer.js +362 -0
  64. package/dist/ner/tokenizer.js.map +1 -0
  65. package/dist/opencode-plugin/git-identity.d.ts +3 -0
  66. package/dist/opencode-plugin/git-identity.d.ts.map +1 -0
  67. package/dist/opencode-plugin/git-identity.js +37 -0
  68. package/dist/opencode-plugin/git-identity.js.map +1 -0
  69. package/dist/opencode-plugin/github-username.d.ts +3 -0
  70. package/dist/opencode-plugin/github-username.d.ts.map +1 -0
  71. package/dist/opencode-plugin/github-username.js +41 -0
  72. package/dist/opencode-plugin/github-username.js.map +1 -0
  73. package/dist/opencode-plugin/identity-recognizer.d.ts +5 -0
  74. package/dist/opencode-plugin/identity-recognizer.d.ts.map +1 -0
  75. package/dist/opencode-plugin/identity-recognizer.js +21 -0
  76. package/dist/opencode-plugin/identity-recognizer.js.map +1 -0
  77. package/dist/opencode-plugin/index.d.ts +7 -0
  78. package/dist/opencode-plugin/index.d.ts.map +1 -0
  79. package/dist/opencode-plugin/index.js +6 -0
  80. package/dist/opencode-plugin/index.js.map +1 -0
  81. package/dist/opencode-plugin/plugin.d.ts +39 -0
  82. package/dist/opencode-plugin/plugin.d.ts.map +1 -0
  83. package/dist/opencode-plugin/plugin.js +294 -0
  84. package/dist/opencode-plugin/plugin.js.map +1 -0
  85. package/dist/opencode-plugin/types.d.ts +33 -0
  86. package/dist/opencode-plugin/types.d.ts.map +1 -0
  87. package/dist/opencode-plugin/types.js +5 -0
  88. package/dist/opencode-plugin/types.js.map +1 -0
  89. package/dist/opencode-plugin/v2/anon-state.d.ts +12 -0
  90. package/dist/opencode-plugin/v2/anon-state.d.ts.map +1 -0
  91. package/dist/opencode-plugin/v2/anon-state.js +14 -0
  92. package/dist/opencode-plugin/v2/anon-state.js.map +1 -0
  93. package/dist/opencode-plugin/v2/deep-rehydrate.d.ts +13 -0
  94. package/dist/opencode-plugin/v2/deep-rehydrate.d.ts.map +1 -0
  95. package/dist/opencode-plugin/v2/deep-rehydrate.js +33 -0
  96. package/dist/opencode-plugin/v2/deep-rehydrate.js.map +1 -0
  97. package/dist/opencode-plugin/v2/index.d.ts +17 -0
  98. package/dist/opencode-plugin/v2/index.d.ts.map +1 -0
  99. package/dist/opencode-plugin/v2/index.js +17 -0
  100. package/dist/opencode-plugin/v2/index.js.map +1 -0
  101. package/dist/opencode-plugin/v2/logger.d.ts +24 -0
  102. package/dist/opencode-plugin/v2/logger.d.ts.map +1 -0
  103. package/dist/opencode-plugin/v2/logger.js +73 -0
  104. package/dist/opencode-plugin/v2/logger.js.map +1 -0
  105. package/dist/opencode-plugin/v2/options.d.ts +63 -0
  106. package/dist/opencode-plugin/v2/options.d.ts.map +1 -0
  107. package/dist/opencode-plugin/v2/options.js +183 -0
  108. package/dist/opencode-plugin/v2/options.js.map +1 -0
  109. package/dist/opencode-plugin/v2/register-hooks.d.ts +43 -0
  110. package/dist/opencode-plugin/v2/register-hooks.d.ts.map +1 -0
  111. package/dist/opencode-plugin/v2/register-hooks.js +160 -0
  112. package/dist/opencode-plugin/v2/register-hooks.js.map +1 -0
  113. package/dist/opencode-plugin/v2/rehydrate-tail.d.ts +101 -0
  114. package/dist/opencode-plugin/v2/rehydrate-tail.d.ts.map +1 -0
  115. package/dist/opencode-plugin/v2/rehydrate-tail.js +147 -0
  116. package/dist/opencode-plugin/v2/rehydrate-tail.js.map +1 -0
  117. package/dist/opencode-plugin/v2/response-rehydrate.d.ts +78 -0
  118. package/dist/opencode-plugin/v2/response-rehydrate.d.ts.map +1 -0
  119. package/dist/opencode-plugin/v2/response-rehydrate.js +210 -0
  120. package/dist/opencode-plugin/v2/response-rehydrate.js.map +1 -0
  121. package/dist/opencode-plugin/v2/scrub-messages.d.ts +39 -0
  122. package/dist/opencode-plugin/v2/scrub-messages.d.ts.map +1 -0
  123. package/dist/opencode-plugin/v2/scrub-messages.js +260 -0
  124. package/dist/opencode-plugin/v2/scrub-messages.js.map +1 -0
  125. package/dist/opencode-plugin/v2/sessions.d.ts +17 -0
  126. package/dist/opencode-plugin/v2/sessions.d.ts.map +1 -0
  127. package/dist/opencode-plugin/v2/sessions.js +23 -0
  128. package/dist/opencode-plugin/v2/sessions.js.map +1 -0
  129. package/dist/opencode-plugin/v2/sse.d.ts +101 -0
  130. package/dist/opencode-plugin/v2/sse.d.ts.map +1 -0
  131. package/dist/opencode-plugin/v2/sse.js +235 -0
  132. package/dist/opencode-plugin/v2/sse.js.map +1 -0
  133. package/dist/opencode-plugin/v2/types.d.ts +188 -0
  134. package/dist/opencode-plugin/v2/types.d.ts.map +1 -0
  135. package/dist/opencode-plugin/v2/types.js +10 -0
  136. package/dist/opencode-plugin/v2/types.js.map +1 -0
  137. package/dist/opencode-plugin/vcs-command.d.ts +3 -0
  138. package/dist/opencode-plugin/vcs-command.d.ts.map +1 -0
  139. package/dist/opencode-plugin/vcs-command.js +34 -0
  140. package/dist/opencode-plugin/vcs-command.js.map +1 -0
  141. package/dist/pipeline/index.d.ts +12 -0
  142. package/dist/pipeline/index.d.ts.map +1 -0
  143. package/dist/pipeline/index.js +12 -0
  144. package/dist/pipeline/index.js.map +1 -0
  145. package/dist/pipeline/prenormalize.d.ts +48 -0
  146. package/dist/pipeline/prenormalize.d.ts.map +1 -0
  147. package/dist/pipeline/prenormalize.js +94 -0
  148. package/dist/pipeline/prenormalize.js.map +1 -0
  149. package/dist/pipeline/resolver.d.ts +56 -0
  150. package/dist/pipeline/resolver.d.ts.map +1 -0
  151. package/dist/pipeline/resolver.js +239 -0
  152. package/dist/pipeline/resolver.js.map +1 -0
  153. package/dist/pipeline/semantic-data-loader.d.ts +171 -0
  154. package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
  155. package/dist/pipeline/semantic-data-loader.js +677 -0
  156. package/dist/pipeline/semantic-data-loader.js.map +1 -0
  157. package/dist/pipeline/semantic-enricher.d.ts +112 -0
  158. package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
  159. package/dist/pipeline/semantic-enricher.js +322 -0
  160. package/dist/pipeline/semantic-enricher.js.map +1 -0
  161. package/dist/pipeline/tagger.d.ts +121 -0
  162. package/dist/pipeline/tagger.d.ts.map +1 -0
  163. package/dist/pipeline/tagger.js +545 -0
  164. package/dist/pipeline/tagger.js.map +1 -0
  165. package/dist/pipeline/title-extractor.d.ts +79 -0
  166. package/dist/pipeline/title-extractor.d.ts.map +1 -0
  167. package/dist/pipeline/title-extractor.js +801 -0
  168. package/dist/pipeline/title-extractor.js.map +1 -0
  169. package/dist/pipeline/validator.d.ts +65 -0
  170. package/dist/pipeline/validator.d.ts.map +1 -0
  171. package/dist/pipeline/validator.js +292 -0
  172. package/dist/pipeline/validator.js.map +1 -0
  173. package/dist/proxy/index.d.ts +13 -0
  174. package/dist/proxy/index.d.ts.map +1 -0
  175. package/dist/proxy/index.js +12 -0
  176. package/dist/proxy/index.js.map +1 -0
  177. package/dist/proxy/providers/anthropic.d.ts +28 -0
  178. package/dist/proxy/providers/anthropic.d.ts.map +1 -0
  179. package/dist/proxy/providers/anthropic.js +265 -0
  180. package/dist/proxy/providers/anthropic.js.map +1 -0
  181. package/dist/proxy/providers/index.d.ts +19 -0
  182. package/dist/proxy/providers/index.d.ts.map +1 -0
  183. package/dist/proxy/providers/index.js +41 -0
  184. package/dist/proxy/providers/index.js.map +1 -0
  185. package/dist/proxy/providers/openai.d.ts +27 -0
  186. package/dist/proxy/providers/openai.d.ts.map +1 -0
  187. package/dist/proxy/providers/openai.js +203 -0
  188. package/dist/proxy/providers/openai.js.map +1 -0
  189. package/dist/proxy/providers/streaming-tool-loop.d.ts +12 -0
  190. package/dist/proxy/providers/streaming-tool-loop.d.ts.map +1 -0
  191. package/dist/proxy/providers/streaming-tool-loop.js +223 -0
  192. package/dist/proxy/providers/streaming-tool-loop.js.map +1 -0
  193. package/dist/proxy/providers/types.d.ts +109 -0
  194. package/dist/proxy/providers/types.d.ts.map +1 -0
  195. package/dist/proxy/providers/types.js +2 -0
  196. package/dist/proxy/providers/types.js.map +1 -0
  197. package/dist/proxy/proxy-server.d.ts +61 -0
  198. package/dist/proxy/proxy-server.d.ts.map +1 -0
  199. package/dist/proxy/proxy-server.js +146 -0
  200. package/dist/proxy/proxy-server.js.map +1 -0
  201. package/dist/proxy/rehydra-fetch.d.ts +35 -0
  202. package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
  203. package/dist/proxy/rehydra-fetch.js +631 -0
  204. package/dist/proxy/rehydra-fetch.js.map +1 -0
  205. package/dist/proxy/rehydra-proxy.d.ts +40 -0
  206. package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
  207. package/dist/proxy/rehydra-proxy.js +150 -0
  208. package/dist/proxy/rehydra-proxy.js.map +1 -0
  209. package/dist/proxy/sse-parser.d.ts +59 -0
  210. package/dist/proxy/sse-parser.d.ts.map +1 -0
  211. package/dist/proxy/sse-parser.js +112 -0
  212. package/dist/proxy/sse-parser.js.map +1 -0
  213. package/dist/proxy/system-instruction.d.ts +19 -0
  214. package/dist/proxy/system-instruction.d.ts.map +1 -0
  215. package/dist/proxy/system-instruction.js +33 -0
  216. package/dist/proxy/system-instruction.js.map +1 -0
  217. package/dist/proxy/tool-loop-stream.d.ts +7 -0
  218. package/dist/proxy/tool-loop-stream.d.ts.map +1 -0
  219. package/dist/proxy/tool-loop-stream.js +65 -0
  220. package/dist/proxy/tool-loop-stream.js.map +1 -0
  221. package/dist/proxy/types.d.ts +138 -0
  222. package/dist/proxy/types.d.ts.map +1 -0
  223. package/dist/proxy/types.js +5 -0
  224. package/dist/proxy/types.js.map +1 -0
  225. package/dist/proxy/wrap-client.d.ts +47 -0
  226. package/dist/proxy/wrap-client.d.ts.map +1 -0
  227. package/dist/proxy/wrap-client.js +70 -0
  228. package/dist/proxy/wrap-client.js.map +1 -0
  229. package/dist/recognizers/address.d.ts +2 -0
  230. package/dist/recognizers/address.d.ts.map +1 -0
  231. package/dist/recognizers/address.js +23 -0
  232. package/dist/recognizers/address.js.map +1 -0
  233. package/dist/recognizers/base.d.ts +78 -0
  234. package/dist/recognizers/base.d.ts.map +1 -0
  235. package/dist/recognizers/base.js +100 -0
  236. package/dist/recognizers/base.js.map +1 -0
  237. package/dist/recognizers/bic-swift.d.ts +10 -0
  238. package/dist/recognizers/bic-swift.d.ts.map +1 -0
  239. package/dist/recognizers/bic-swift.js +107 -0
  240. package/dist/recognizers/bic-swift.js.map +1 -0
  241. package/dist/recognizers/credit-card.d.ts +32 -0
  242. package/dist/recognizers/credit-card.d.ts.map +1 -0
  243. package/dist/recognizers/credit-card.js +160 -0
  244. package/dist/recognizers/credit-card.js.map +1 -0
  245. package/dist/recognizers/custom-id.d.ts +28 -0
  246. package/dist/recognizers/custom-id.d.ts.map +1 -0
  247. package/dist/recognizers/custom-id.js +116 -0
  248. package/dist/recognizers/custom-id.js.map +1 -0
  249. package/dist/recognizers/date.d.ts +10 -0
  250. package/dist/recognizers/date.d.ts.map +1 -0
  251. package/dist/recognizers/date.js +228 -0
  252. package/dist/recognizers/date.js.map +1 -0
  253. package/dist/recognizers/email.d.ts +10 -0
  254. package/dist/recognizers/email.d.ts.map +1 -0
  255. package/dist/recognizers/email.js +75 -0
  256. package/dist/recognizers/email.js.map +1 -0
  257. package/dist/recognizers/iban.d.ts +14 -0
  258. package/dist/recognizers/iban.d.ts.map +1 -0
  259. package/dist/recognizers/iban.js +67 -0
  260. package/dist/recognizers/iban.js.map +1 -0
  261. package/dist/recognizers/index.d.ts +25 -0
  262. package/dist/recognizers/index.d.ts.map +1 -0
  263. package/dist/recognizers/index.js +53 -0
  264. package/dist/recognizers/index.js.map +1 -0
  265. package/dist/recognizers/ip-address.d.ts +14 -0
  266. package/dist/recognizers/ip-address.d.ts.map +1 -0
  267. package/dist/recognizers/ip-address.js +183 -0
  268. package/dist/recognizers/ip-address.js.map +1 -0
  269. package/dist/recognizers/phone.d.ts +10 -0
  270. package/dist/recognizers/phone.d.ts.map +1 -0
  271. package/dist/recognizers/phone.js +145 -0
  272. package/dist/recognizers/phone.js.map +1 -0
  273. package/dist/recognizers/postal-code.d.ts +16 -0
  274. package/dist/recognizers/postal-code.d.ts.map +1 -0
  275. package/dist/recognizers/postal-code.js +42 -0
  276. package/dist/recognizers/postal-code.js.map +1 -0
  277. package/dist/recognizers/registry.d.ts +59 -0
  278. package/dist/recognizers/registry.d.ts.map +1 -0
  279. package/dist/recognizers/registry.js +113 -0
  280. package/dist/recognizers/registry.js.map +1 -0
  281. package/dist/recognizers/secrets/api-key.d.ts +6 -0
  282. package/dist/recognizers/secrets/api-key.d.ts.map +1 -0
  283. package/dist/recognizers/secrets/api-key.js +40 -0
  284. package/dist/recognizers/secrets/api-key.js.map +1 -0
  285. package/dist/recognizers/secrets/aws-credentials.d.ts +7 -0
  286. package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -0
  287. package/dist/recognizers/secrets/aws-credentials.js +68 -0
  288. package/dist/recognizers/secrets/aws-credentials.js.map +1 -0
  289. package/dist/recognizers/secrets/config-secret.d.ts +8 -0
  290. package/dist/recognizers/secrets/config-secret.d.ts.map +1 -0
  291. package/dist/recognizers/secrets/config-secret.js +59 -0
  292. package/dist/recognizers/secrets/config-secret.js.map +1 -0
  293. package/dist/recognizers/secrets/connection-string.d.ts +6 -0
  294. package/dist/recognizers/secrets/connection-string.d.ts.map +1 -0
  295. package/dist/recognizers/secrets/connection-string.js +40 -0
  296. package/dist/recognizers/secrets/connection-string.js.map +1 -0
  297. package/dist/recognizers/secrets/env-var.d.ts +8 -0
  298. package/dist/recognizers/secrets/env-var.d.ts.map +1 -0
  299. package/dist/recognizers/secrets/env-var.js +54 -0
  300. package/dist/recognizers/secrets/env-var.js.map +1 -0
  301. package/dist/recognizers/secrets/index.d.ts +22 -0
  302. package/dist/recognizers/secrets/index.d.ts.map +1 -0
  303. package/dist/recognizers/secrets/index.js +41 -0
  304. package/dist/recognizers/secrets/index.js.map +1 -0
  305. package/dist/recognizers/secrets/jwt.d.ts +6 -0
  306. package/dist/recognizers/secrets/jwt.d.ts.map +1 -0
  307. package/dist/recognizers/secrets/jwt.js +47 -0
  308. package/dist/recognizers/secrets/jwt.js.map +1 -0
  309. package/dist/recognizers/secrets/key-patterns.d.ts +10 -0
  310. package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -0
  311. package/dist/recognizers/secrets/key-patterns.js +15 -0
  312. package/dist/recognizers/secrets/key-patterns.js.map +1 -0
  313. package/dist/recognizers/secrets/literal-value.d.ts +10 -0
  314. package/dist/recognizers/secrets/literal-value.d.ts.map +1 -0
  315. package/dist/recognizers/secrets/literal-value.js +48 -0
  316. package/dist/recognizers/secrets/literal-value.js.map +1 -0
  317. package/dist/recognizers/secrets/private-key.d.ts +6 -0
  318. package/dist/recognizers/secrets/private-key.d.ts.map +1 -0
  319. package/dist/recognizers/secrets/private-key.js +16 -0
  320. package/dist/recognizers/secrets/private-key.js.map +1 -0
  321. package/dist/recognizers/url.d.ts +14 -0
  322. package/dist/recognizers/url.d.ts.map +1 -0
  323. package/dist/recognizers/url.js +121 -0
  324. package/dist/recognizers/url.js.map +1 -0
  325. package/dist/storage/browser.d.ts +13 -0
  326. package/dist/storage/browser.d.ts.map +1 -0
  327. package/dist/storage/browser.js +13 -0
  328. package/dist/storage/browser.js.map +1 -0
  329. package/dist/storage/in-memory.d.ts +60 -0
  330. package/dist/storage/in-memory.d.ts.map +1 -0
  331. package/dist/storage/in-memory.js +108 -0
  332. package/dist/storage/in-memory.js.map +1 -0
  333. package/dist/storage/index.d.ts +11 -0
  334. package/dist/storage/index.d.ts.map +1 -0
  335. package/dist/storage/index.js +11 -0
  336. package/dist/storage/index.js.map +1 -0
  337. package/dist/storage/indexeddb.d.ts +69 -0
  338. package/dist/storage/indexeddb.d.ts.map +1 -0
  339. package/dist/storage/indexeddb.js +274 -0
  340. package/dist/storage/indexeddb.js.map +1 -0
  341. package/dist/storage/session-base.d.ts +34 -0
  342. package/dist/storage/session-base.d.ts.map +1 -0
  343. package/dist/storage/session-base.js +122 -0
  344. package/dist/storage/session-base.js.map +1 -0
  345. package/dist/storage/session.d.ts +19 -0
  346. package/dist/storage/session.d.ts.map +1 -0
  347. package/dist/storage/session.js +31 -0
  348. package/dist/storage/session.js.map +1 -0
  349. package/dist/storage/sqlite.d.ts +80 -0
  350. package/dist/storage/sqlite.d.ts.map +1 -0
  351. package/dist/storage/sqlite.js +255 -0
  352. package/dist/storage/sqlite.js.map +1 -0
  353. package/dist/storage/types.d.ts +208 -0
  354. package/dist/storage/types.d.ts.map +1 -0
  355. package/dist/storage/types.js +6 -0
  356. package/dist/storage/types.js.map +1 -0
  357. package/dist/streaming/anonymizer-stream.d.ts +63 -0
  358. package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
  359. package/dist/streaming/anonymizer-stream.js +184 -0
  360. package/dist/streaming/anonymizer-stream.js.map +1 -0
  361. package/dist/streaming/index.d.ts +9 -0
  362. package/dist/streaming/index.d.ts.map +1 -0
  363. package/dist/streaming/index.js +8 -0
  364. package/dist/streaming/index.js.map +1 -0
  365. package/dist/streaming/sentence-buffer.d.ts +78 -0
  366. package/dist/streaming/sentence-buffer.d.ts.map +1 -0
  367. package/dist/streaming/sentence-buffer.js +239 -0
  368. package/dist/streaming/sentence-buffer.js.map +1 -0
  369. package/dist/streaming/stream-factory.d.ts +38 -0
  370. package/dist/streaming/stream-factory.d.ts.map +1 -0
  371. package/dist/streaming/stream-factory.js +69 -0
  372. package/dist/streaming/stream-factory.js.map +1 -0
  373. package/dist/streaming/types.d.ts +121 -0
  374. package/dist/streaming/types.d.ts.map +1 -0
  375. package/dist/streaming/types.js +5 -0
  376. package/dist/streaming/types.js.map +1 -0
  377. package/dist/types/index.d.ts +269 -0
  378. package/dist/types/index.d.ts.map +1 -0
  379. package/dist/types/index.js +95 -0
  380. package/dist/types/index.js.map +1 -0
  381. package/dist/types/pii-types.d.ts +64 -0
  382. package/dist/types/pii-types.d.ts.map +1 -0
  383. package/dist/types/pii-types.js +153 -0
  384. package/dist/types/pii-types.js.map +1 -0
  385. package/dist/utils/iban-checksum.d.ts +23 -0
  386. package/dist/utils/iban-checksum.d.ts.map +1 -0
  387. package/dist/utils/iban-checksum.js +106 -0
  388. package/dist/utils/iban-checksum.js.map +1 -0
  389. package/dist/utils/index.d.ts +11 -0
  390. package/dist/utils/index.d.ts.map +1 -0
  391. package/dist/utils/index.js +11 -0
  392. package/dist/utils/index.js.map +1 -0
  393. package/dist/utils/json-walk.d.ts +21 -0
  394. package/dist/utils/json-walk.d.ts.map +1 -0
  395. package/dist/utils/json-walk.js +44 -0
  396. package/dist/utils/json-walk.js.map +1 -0
  397. package/dist/utils/luhn.d.ts +17 -0
  398. package/dist/utils/luhn.d.ts.map +1 -0
  399. package/dist/utils/luhn.js +55 -0
  400. package/dist/utils/luhn.js.map +1 -0
  401. package/dist/utils/offsets.d.ts +86 -0
  402. package/dist/utils/offsets.d.ts.map +1 -0
  403. package/dist/utils/offsets.js +124 -0
  404. package/dist/utils/offsets.js.map +1 -0
  405. package/dist/utils/path.d.ts +34 -0
  406. package/dist/utils/path.d.ts.map +1 -0
  407. package/dist/utils/path.js +96 -0
  408. package/dist/utils/path.js.map +1 -0
  409. package/dist/utils/regex.d.ts +11 -0
  410. package/dist/utils/regex.d.ts.map +1 -0
  411. package/dist/utils/regex.js +14 -0
  412. package/dist/utils/regex.js.map +1 -0
  413. package/dist/utils/storage-browser.d.ts +51 -0
  414. package/dist/utils/storage-browser.d.ts.map +1 -0
  415. package/dist/utils/storage-browser.js +381 -0
  416. package/dist/utils/storage-browser.js.map +1 -0
  417. package/dist/utils/storage-node.d.ts +51 -0
  418. package/dist/utils/storage-node.d.ts.map +1 -0
  419. package/dist/utils/storage-node.js +153 -0
  420. package/dist/utils/storage-node.js.map +1 -0
  421. package/dist/utils/storage.browser.d.ts +75 -0
  422. package/dist/utils/storage.browser.d.ts.map +1 -0
  423. package/dist/utils/storage.browser.js +69 -0
  424. package/dist/utils/storage.browser.js.map +1 -0
  425. package/dist/utils/storage.d.ts +73 -0
  426. package/dist/utils/storage.d.ts.map +1 -0
  427. package/dist/utils/storage.js +69 -0
  428. package/dist/utils/storage.js.map +1 -0
  429. package/index.js +14 -13
  430. package/package.json +19 -5
@@ -0,0 +1,90 @@
1
+ /**
2
+ * ONNX Runtime Abstraction - Browser-Only Version
3
+ *
4
+ * This module provides the same interface as onnx-runtime.ts but only includes
5
+ * onnxruntime-web. It's used by the browser entry point to avoid bundler issues
6
+ * with onnxruntime-node.
7
+ *
8
+ * DO NOT import onnxruntime-node here - that's the whole point!
9
+ */
10
+ // CDN URL for onnxruntime-web (used when bare import fails in browser)
11
+ // Using the bundled ESM version that includes WebAssembly backend
12
+ const ONNX_WEB_CDN_URL = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/ort.bundle.min.mjs";
13
+ /**
14
+ * Runtime detection and loading
15
+ */
16
+ let _runtime = null;
17
+ let _runtimeType = null;
18
+ /**
19
+ * Detects the best ONNX runtime for the current environment
20
+ * Browser version always returns "web"
21
+ */
22
+ export function detectRuntime() {
23
+ return "web";
24
+ }
25
+ /**
26
+ * Attempts to load onnxruntime-web, first via bare import, then via CDN
27
+ */
28
+ async function loadOnnxWeb() {
29
+ // First try bare import (works with bundlers or import maps)
30
+ try {
31
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
32
+ // @ts-ignore - onnxruntime-web may not be installed
33
+ const ort = (await import("onnxruntime-web"));
34
+ return ort;
35
+ }
36
+ catch {
37
+ // Bare import failed, try CDN (for browsers without bundlers)
38
+ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
39
+ const isBrowser = typeof globalThis.window !== "undefined";
40
+ if (isBrowser) {
41
+ try {
42
+ // Dynamic import from CDN URL
43
+ const ort = (await import(
44
+ /* webpackIgnore: true */ ONNX_WEB_CDN_URL));
45
+ return ort;
46
+ }
47
+ catch (cdnError) {
48
+ throw new Error(`Failed to load onnxruntime-web from CDN: ${String(cdnError)}`);
49
+ }
50
+ }
51
+ throw new Error("onnxruntime-web is not available");
52
+ }
53
+ }
54
+ /**
55
+ * Loads the ONNX runtime (always onnxruntime-web in browser build)
56
+ */
57
+ export async function loadRuntime(preferredRuntime) {
58
+ if (_runtime !== null) {
59
+ return _runtime;
60
+ }
61
+ // In browser build, we only support "web" runtime
62
+ if (preferredRuntime === "node") {
63
+ throw new Error("onnxruntime-node is not available in browser build.\n" +
64
+ "Import from 'rehydra' (not 'rehydra/browser') for Node.js support.");
65
+ }
66
+ try {
67
+ const ort = await loadOnnxWeb();
68
+ _runtime = ort;
69
+ _runtimeType = "web";
70
+ }
71
+ catch (e) {
72
+ throw new Error(`Failed to load ONNX runtime. Make sure 'onnxruntime-web' is installed.\n` +
73
+ `Original error: ${String(e)}`);
74
+ }
75
+ return _runtime;
76
+ }
77
+ /**
78
+ * Gets the currently loaded runtime type
79
+ */
80
+ export function getRuntimeType() {
81
+ return _runtimeType;
82
+ }
83
+ /**
84
+ * Resets the runtime (useful for testing)
85
+ */
86
+ export function resetRuntime() {
87
+ _runtime = null;
88
+ _runtimeType = null;
89
+ }
90
+ //# sourceMappingURL=onnx-runtime.browser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"onnx-runtime.browser.js","sourceRoot":"","sources":["../../src/ner/onnx-runtime.browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,uEAAuE;AACvE,kEAAkE;AAClE,MAAM,gBAAgB,GACpB,6EAA6E,CAAC;AAkChF;;GAEG;AACH,IAAI,QAAQ,GAAsB,IAAI,CAAC;AACvC,IAAI,YAAY,GAA0B,IAAI,CAAC;AAE/C;;;GAGG;AACH,MAAM,UAAU,aAAa;IAC3B,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,6DAA6D;IAC7D,IAAI,CAAC;QACH,6DAA6D;QAC7D,oDAAoD;QACpD,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAe,CAAC;QAC5D,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,uEAAuE;QACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;QAC3D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,8BAA8B;gBAC9B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM;gBACvB,yBAAyB,CAAC,gBAAgB,CAC3C,CAAe,CAAC;gBACjB,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAC/D,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,gBAAiC;IAEjC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,kDAAkD;IAClD,IAAI,gBAAgB,KAAK,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,uDAAuD;YACvD,oEAAoE,CACrE,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;QAChC,QAAQ,GAAG,GAAG,CAAC;QACf,YAAY,GAAG,KAAK,CAAC;IACvB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,mBAAmB,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,QAAQ,GAAG,IAAI,CAAC;IAChB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
@@ -0,0 +1,72 @@
1
+ /**
2
+ * ONNX Runtime Abstraction
3
+ * Allows switching between onnxruntime-node and onnxruntime-web
4
+ *
5
+ * In browsers without a bundler, automatically loads onnxruntime-web from CDN
6
+ */
7
+ export interface OrtTensor {
8
+ data: Float32Array | BigInt64Array | Int32Array;
9
+ dims: readonly number[];
10
+ }
11
+ export interface OrtSession {
12
+ inputNames: readonly string[];
13
+ outputNames: readonly string[];
14
+ run(feeds: Record<string, OrtTensor>): Promise<Record<string, OrtTensor>>;
15
+ }
16
+ /**
17
+ * ONNX Runtime session options for performance tuning
18
+ */
19
+ export interface OrtSessionOptions {
20
+ /**
21
+ * Execution providers in priority order.
22
+ * - For onnxruntime-web: ['webgpu', 'wasm'] or ['wasm']
23
+ * - For onnxruntime-node: usually not needed (uses CPU by default)
24
+ * - With custom builds: ['coreml'], etc.
25
+ */
26
+ executionProviders?: Array<string | {
27
+ name: string;
28
+ [key: string]: unknown;
29
+ }>;
30
+ /** Graph optimization level */
31
+ graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";
32
+ /** Number of threads for parallel execution within operators */
33
+ intraOpNumThreads?: number;
34
+ /** Number of threads for parallel execution between operators */
35
+ interOpNumThreads?: number;
36
+ /** Enable CPU memory arena for allocations */
37
+ enableCpuMemArena?: boolean;
38
+ /** Enable memory pattern optimization */
39
+ enableMemPattern?: boolean;
40
+ }
41
+ export interface OrtInferenceSession {
42
+ create(pathOrBuffer: string | ArrayBuffer | Uint8Array, options?: OrtSessionOptions): Promise<OrtSession>;
43
+ }
44
+ export interface OrtTensorConstructor {
45
+ new (type: string, data: Float32Array | BigInt64Array | Int32Array | number[] | bigint[], dims: number[]): OrtTensor;
46
+ }
47
+ export interface OrtRuntime {
48
+ InferenceSession: OrtInferenceSession;
49
+ Tensor: OrtTensorConstructor;
50
+ }
51
+ /**
52
+ * Detects the best ONNX runtime for the current environment
53
+ */
54
+ export declare function detectRuntime(): "node" | "web";
55
+ /**
56
+ * Loads the appropriate ONNX runtime
57
+ * @param preferredRuntime - Force 'node' or 'web' runtime
58
+ */
59
+ export declare function loadRuntime(preferredRuntime?: "node" | "web"): Promise<OrtRuntime>;
60
+ /**
61
+ * Gets the currently loaded runtime type
62
+ */
63
+ export declare function getRuntimeType(): "node" | "web" | null;
64
+ /**
65
+ * Resets the runtime (useful for testing)
66
+ */
67
+ export declare function resetRuntime(): void;
68
+ declare global {
69
+ var Bun: unknown;
70
+ var Deno: unknown;
71
+ }
72
+ //# sourceMappingURL=onnx-runtime.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"onnx-runtime.d.ts","sourceRoot":"","sources":["../../src/ner/onnx-runtime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,UAAU,CAAC;IAChD,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;CAC3E;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAC;IAC9E,+BAA+B;IAC/B,sBAAsB,CAAC,EAAE,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,KAAK,CAAC;IACnE,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8CAA8C;IAC9C,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,CACJ,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,UAAU,EAC/C,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,KACE,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,UAAU,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,EACrE,IAAI,EAAE,MAAM,EAAE,GACb,SAAS,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,gBAAgB,EAAE,mBAAmB,CAAC;IACtC,MAAM,EAAE,oBAAoB,CAAC;CAC9B;AAQD;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,KAAK,CA4B9C;AAiCD;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,gBAAgB,CAAC,EAAE,MAAM,GAAG,KAAK,GAChC,OAAO,CAAC,UAAU,CAAC,CA4CrB;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,GAAG,KAAK,GAAG,IAAI,CAEtD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAGnC;AAGD,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,GAAG,EAAE,OAAO,CAAC;IAEjB,IAAI,IAAI,EAAE,OAAO,CAAC;CACnB"}
@@ -0,0 +1,132 @@
1
+ /**
2
+ * ONNX Runtime Abstraction
3
+ * Allows switching between onnxruntime-node and onnxruntime-web
4
+ *
5
+ * In browsers without a bundler, automatically loads onnxruntime-web from CDN
6
+ */
7
+ // CDN URL for onnxruntime-web (used when bare import fails in browser)
8
+ // Using the bundled ESM version that includes WebAssembly backend
9
+ const ONNX_WEB_CDN_URL = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/ort.bundle.min.mjs";
10
+ /**
11
+ * Runtime detection and loading
12
+ */
13
+ let _runtime = null;
14
+ let _runtimeType = null;
15
+ /**
16
+ * Detects the best ONNX runtime for the current environment
17
+ */
18
+ export function detectRuntime() {
19
+ // Check if we're in Bun
20
+ const isBun = typeof globalThis.Bun !== "undefined";
21
+ // Check if we're in a browser-like environment
22
+ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
23
+ const isBrowser = typeof globalThis.window !== "undefined";
24
+ // Check if we're in Deno
25
+ const isDeno = typeof globalThis.Deno !== "undefined";
26
+ if (isBrowser || isDeno) {
27
+ return "web";
28
+ }
29
+ // For Bun, try node first, fall back to web
30
+ if (isBun) {
31
+ try {
32
+ // Quick check if onnxruntime-node is loadable
33
+ require.resolve("onnxruntime-node");
34
+ return "node";
35
+ }
36
+ catch {
37
+ return "web";
38
+ }
39
+ }
40
+ // Default to node for Node.js
41
+ return "node";
42
+ }
43
+ /**
44
+ * Attempts to load onnxruntime-web, first via bare import, then via CDN
45
+ */
46
+ async function loadOnnxWeb() {
47
+ // First try bare import (works with bundlers or import maps)
48
+ try {
49
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
50
+ // @ts-ignore - onnxruntime-web may not be installed
51
+ const ort = (await import("onnxruntime-web"));
52
+ return ort;
53
+ }
54
+ catch {
55
+ // Bare import failed, try CDN (for browsers without bundlers)
56
+ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
57
+ const isBrowser = typeof globalThis.window !== "undefined";
58
+ if (isBrowser) {
59
+ try {
60
+ // Dynamic import from CDN URL
61
+ const ort = (await import(
62
+ /* webpackIgnore: true */ ONNX_WEB_CDN_URL));
63
+ return ort;
64
+ }
65
+ catch (cdnError) {
66
+ throw new Error(`Failed to load onnxruntime-web from CDN: ${String(cdnError)}`);
67
+ }
68
+ }
69
+ throw new Error("onnxruntime-web is not available");
70
+ }
71
+ }
72
+ /**
73
+ * Loads the appropriate ONNX runtime
74
+ * @param preferredRuntime - Force 'node' or 'web' runtime
75
+ */
76
+ export async function loadRuntime(preferredRuntime) {
77
+ // If runtime is already loaded, return it
78
+ if (_runtime !== null) {
79
+ return _runtime;
80
+ }
81
+ const runtimeType = preferredRuntime ?? detectRuntime();
82
+ try {
83
+ if (runtimeType === "node") {
84
+ // Load standard CPU runtime (works with Bun)
85
+ const ort = (await import("onnxruntime-node"));
86
+ _runtime = ort;
87
+ _runtimeType = "node";
88
+ }
89
+ else {
90
+ // Load onnxruntime-web (with CDN fallback for browsers)
91
+ const ort = await loadOnnxWeb();
92
+ _runtime = ort;
93
+ _runtimeType = "web";
94
+ }
95
+ }
96
+ catch (e) {
97
+ // If preferred runtime fails, try the other
98
+ const fallbackType = runtimeType === "node" ? "web" : "node";
99
+ try {
100
+ if (fallbackType === "node") {
101
+ const ort = (await import("onnxruntime-node"));
102
+ _runtime = ort;
103
+ _runtimeType = "node";
104
+ }
105
+ else {
106
+ // Load onnxruntime-web (with CDN fallback for browsers)
107
+ const ort = await loadOnnxWeb();
108
+ _runtime = ort;
109
+ _runtimeType = "web";
110
+ }
111
+ }
112
+ catch {
113
+ throw new Error(`Failed to load ONNX runtime. Install either 'onnxruntime-node' or 'onnxruntime-web'.\n` +
114
+ `Original error: ${String(e)}`);
115
+ }
116
+ }
117
+ return _runtime;
118
+ }
119
+ /**
120
+ * Gets the currently loaded runtime type
121
+ */
122
+ export function getRuntimeType() {
123
+ return _runtimeType;
124
+ }
125
+ /**
126
+ * Resets the runtime (useful for testing)
127
+ */
128
+ export function resetRuntime() {
129
+ _runtime = null;
130
+ _runtimeType = null;
131
+ }
132
+ //# sourceMappingURL=onnx-runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"onnx-runtime.js","sourceRoot":"","sources":["../../src/ner/onnx-runtime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,uEAAuE;AACvE,kEAAkE;AAClE,MAAM,gBAAgB,GACpB,6EAA6E,CAAC;AAyDhF;;GAEG;AACH,IAAI,QAAQ,GAAsB,IAAI,CAAC;AACvC,IAAI,YAAY,GAA0B,IAAI,CAAC;AAE/C;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,wBAAwB;IACxB,MAAM,KAAK,GAAG,OAAO,UAAU,CAAC,GAAG,KAAK,WAAW,CAAC;IAEpD,+CAA+C;IAC/C,uEAAuE;IACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;IAE3D,yBAAyB;IACzB,MAAM,MAAM,GAAG,OAAO,UAAU,CAAC,IAAI,KAAK,WAAW,CAAC;IAEtD,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,4CAA4C;IAC5C,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC;YACH,8CAA8C;YAC9C,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,6DAA6D;IAC7D,IAAI,CAAC;QACH,6DAA6D;QAC7D,oDAAoD;QACpD,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAe,CAAC;QAC5D,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,uEAAuE;QACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;QAC3D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,8BAA8B;gBAC9B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM;gBACvB,yBAAyB,CAAC,gBAAgB,CAC3C,CAAe,CAAC;gBACjB,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAC/D,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,gBAAiC;IAEjC,0CAA0C;IAC1C,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,gBAAgB,IAAI,aAAa,EAAE,CAAC;IAExD,IAAI,CAAC;QACH,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,6CAA6C;YAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAe,CAAC;YAC7D,QAAQ,GAAG,GAAG,CAAC;YACf,YAAY,GAAG,MAAM,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;YAChC,QAAQ,GAAG,GAAG,CAAC;YACf,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,4CAA4C;QAC5C,MAAM,YAAY,GAAG,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QAE7D,IAAI,CAAC;YACH,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAe,CAAC;gBAC7D,QAAQ,GAAG,GAAG,CAAC;gBACf,YAAY,GAAG,MAAM,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,wDAAwD;gBACxD,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;gBAChC,QAAQ,GAAG,GAAG,CAAC;gBACf,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,wFAAwF;gBACtF,mBAAmB,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,QAAQ,GAAG,IAAI,CAAC;IAChB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
@@ -0,0 +1,19 @@
1
+ import type { SpanMatch } from "../types/index.js";
2
+ import type { TokenizationResult } from "./tokenizer.js";
3
+ /**
4
+ * Total token count (including the two special tokens) that `maxWindows`
5
+ * windows can consume. Used to bound tokenization up front so huge inputs
6
+ * are not fully tokenized just to discard everything past the cap.
7
+ */
8
+ export declare function windowTokenBudget(maxLength: number, overlapTokens: number, maxWindows: number): number;
9
+ export declare function validateWindowConfig(maxLength: number, overlapTokens: number, maxWindows?: number): void;
10
+ /**
11
+ * Splits an already-tokenized input into overlapping model-sized windows.
12
+ * The window count is bounded by the caller pre-bounding tokenization via
13
+ * `tokenize`'s `maxTokens` (see `runNERPass`) — this just slices whatever it
14
+ * is handed, so a truncated tokenization yields a truncated set of windows,
15
+ * and the truncation signal lives on the parent `TokenizationResult`.
16
+ */
17
+ export declare function createTokenWindows(tokenization: TokenizationResult, maxLength: number, overlapTokens: number): TokenizationResult[];
18
+ export declare function mergeWindowSpans(spans: SpanMatch[], originalText: string): SpanMatch[];
19
+ //# sourceMappingURL=token-window.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"token-window.d.ts","sourceRoot":"","sources":["../../src/ner/token-window.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAS,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAiBhE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,GACjB,MAAM,CAKR;AAED,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,CAAC,EAAE,MAAM,GAClB,IAAI,CAUN;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,YAAY,EAAE,kBAAkB,EAChC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,GACpB,kBAAkB,EAAE,CA+BtB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAwCb"}
@@ -0,0 +1,101 @@
1
+ function buildTokenization(tokens) {
2
+ return {
3
+ tokens,
4
+ inputIds: tokens.map((token) => token.id),
5
+ attentionMask: tokens.map(() => 1),
6
+ tokenTypeIds: tokens.map(() => 0),
7
+ tokenToCharSpan: tokens.map((token) => token.isSpecial ? null : [token.start, token.end]),
8
+ // A window is a complete slice of an already-tokenized input; the
9
+ // truncated signal lives on the parent tokenization, not each window.
10
+ truncated: false,
11
+ };
12
+ }
13
+ /**
14
+ * Total token count (including the two special tokens) that `maxWindows`
15
+ * windows can consume. Used to bound tokenization up front so huge inputs
16
+ * are not fully tokenized just to discard everything past the cap.
17
+ */
18
+ export function windowTokenBudget(maxLength, overlapTokens, maxWindows) {
19
+ validateWindowConfig(maxLength, overlapTokens, maxWindows);
20
+ const contentCapacity = maxLength - 2;
21
+ const stride = contentCapacity - overlapTokens;
22
+ return 2 + contentCapacity + (maxWindows - 1) * stride;
23
+ }
24
+ export function validateWindowConfig(maxLength, overlapTokens, maxWindows) {
25
+ if (maxLength - 2 < 1) {
26
+ throw new Error("NER maxLength must leave room for content and two special tokens");
27
+ }
28
+ if (overlapTokens < 0 || overlapTokens >= maxLength - 2) {
29
+ throw new Error("NER window overlap must be smaller than the content capacity");
30
+ }
31
+ if (maxWindows !== undefined && maxWindows < 1) {
32
+ throw new Error("NER maxWindowsPerInput must be at least 1");
33
+ }
34
+ }
35
+ /**
36
+ * Splits an already-tokenized input into overlapping model-sized windows.
37
+ * The window count is bounded by the caller pre-bounding tokenization via
38
+ * `tokenize`'s `maxTokens` (see `runNERPass`) — this just slices whatever it
39
+ * is handed, so a truncated tokenization yields a truncated set of windows,
40
+ * and the truncation signal lives on the parent `TokenizationResult`.
41
+ */
42
+ export function createTokenWindows(tokenization, maxLength, overlapTokens) {
43
+ validateWindowConfig(maxLength, overlapTokens);
44
+ const contentCapacity = maxLength - 2;
45
+ if (tokenization.tokens.length <= maxLength) {
46
+ return [tokenization];
47
+ }
48
+ const firstToken = tokenization.tokens[0];
49
+ const lastToken = tokenization.tokens[tokenization.tokens.length - 1];
50
+ if (firstToken === undefined ||
51
+ firstToken.isSpecial !== true ||
52
+ lastToken === undefined ||
53
+ lastToken.isSpecial !== true) {
54
+ throw new Error("NER tokenization must include leading and trailing special tokens");
55
+ }
56
+ const contentTokens = tokenization.tokens.slice(1, -1);
57
+ const stride = contentCapacity - overlapTokens;
58
+ const windows = [];
59
+ for (let start = 0; start < contentTokens.length; start += stride) {
60
+ const end = Math.min(start + contentCapacity, contentTokens.length);
61
+ windows.push(buildTokenization([firstToken, ...contentTokens.slice(start, end), lastToken]));
62
+ if (end === contentTokens.length)
63
+ break;
64
+ }
65
+ return windows;
66
+ }
67
+ export function mergeWindowSpans(spans, originalText) {
68
+ const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - a.end || b.confidence - a.confidence);
69
+ const merged = [];
70
+ for (const span of sorted) {
71
+ let combined = span;
72
+ for (let index = merged.length - 1; index >= 0; index--) {
73
+ const existing = merged[index];
74
+ // Only stitch spans that actually overlap (touching end-to-start is not
75
+ // an overlap). This relies on the window overlap being at least as long
76
+ // as the longest expected entity — otherwise an entity split across a
77
+ // window boundary produces two abutting fragments that never merge.
78
+ // The default overlap (64 tokens) is far longer than any name/entity.
79
+ if (existing.type !== combined.type ||
80
+ existing.start >= combined.end ||
81
+ combined.start >= existing.end) {
82
+ continue;
83
+ }
84
+ const preferred = combined.confidence > existing.confidence
85
+ ? combined
86
+ : existing;
87
+ const start = Math.min(existing.start, combined.start);
88
+ const end = Math.max(existing.end, combined.end);
89
+ combined = {
90
+ ...preferred,
91
+ start,
92
+ end,
93
+ text: originalText.slice(start, end),
94
+ };
95
+ merged.splice(index, 1);
96
+ }
97
+ merged.push(combined);
98
+ }
99
+ return merged.sort((a, b) => a.start - b.start || b.end - a.end);
100
+ }
101
+ //# sourceMappingURL=token-window.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"token-window.js","sourceRoot":"","sources":["../../src/ner/token-window.ts"],"names":[],"mappings":"AAGA,SAAS,iBAAiB,CAAC,MAAe;IACxC,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACjC,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACpC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAClD;QACD,kEAAkE;QAClE,sEAAsE;QACtE,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAAiB,EACjB,aAAqB,EACrB,UAAkB;IAElB,oBAAoB,CAAC,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,eAAe,GAAG,aAAa,CAAC;IAC/C,OAAO,CAAC,GAAG,eAAe,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,SAAiB,EACjB,aAAqB,EACrB,UAAmB;IAEnB,IAAI,SAAS,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,aAAa,GAAG,CAAC,IAAI,aAAa,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,YAAgC,EAChC,SAAiB,EACjB,aAAqB;IAErB,oBAAoB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;IACtC,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC5C,OAAO,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtE,IACE,UAAU,KAAK,SAAS;QACxB,UAAU,CAAC,SAAS,KAAK,IAAI;QAC7B,SAAS,KAAK,SAAS;QACvB,SAAS,CAAC,SAAS,KAAK,IAAI,EAC5B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,eAAe,GAAG,aAAa,CAAC;IAC/C,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,eAAe,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CACV,iBAAiB,CAAC,CAAC,UAAU,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAC,CAC/E,CAAC;QACF,IAAI,GAAG,KAAK,aAAa,CAAC,MAAM;YAAE,MAAM;IAC1C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,KAAkB,EAClB,YAAoB;IAEpB,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAClE,CAAC;IACF,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,KAAK,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;YAChC,wEAAwE;YACxE,wEAAwE;YACxE,sEAAsE;YACtE,oEAAoE;YACpE,sEAAsE;YACtE,IACE,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;gBAC/B,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG;gBAC9B,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,EAC9B,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU;gBACzD,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACjD,QAAQ,GAAG;gBACT,GAAG,SAAS;gBACZ,KAAK;gBACL,GAAG;gBACH,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;aACrC,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACnE,CAAC"}
@@ -0,0 +1,142 @@
1
+ /**
2
+ * HuggingFace Tokenizer
3
+ * Loads and uses tokenizers from HuggingFace's tokenizer.json format
4
+ * Supports Unigram (SentencePiece) and BPE tokenizers
5
+ */
6
+ /**
7
+ * Token with offset information
8
+ */
9
+ export interface Token {
10
+ /** Token ID in vocabulary */
11
+ id: number;
12
+ /** Token string */
13
+ token: string;
14
+ /** Start character offset in original text */
15
+ start: number;
16
+ /** End character offset in original text */
17
+ end: number;
18
+ /** Whether this is a continuation token */
19
+ isContinuation: boolean;
20
+ /** Whether this is a special token */
21
+ isSpecial: boolean;
22
+ }
23
+ /**
24
+ * Tokenization result with metadata
25
+ */
26
+ export interface TokenizationResult {
27
+ /** Array of tokens */
28
+ tokens: Token[];
29
+ /** Input IDs for model */
30
+ inputIds: number[];
31
+ /** Attention mask */
32
+ attentionMask: number[];
33
+ /** Token type IDs (for BERT-style models) */
34
+ tokenTypeIds: number[];
35
+ /** Mapping from token index to character span [start, end] */
36
+ tokenToCharSpan: Array<[number, number] | null>;
37
+ /**
38
+ * True when tokenization stopped before consuming all input — either
39
+ * because `truncate` capped it at maxLength or because `maxTokens` bounded
40
+ * it. Lets callers know detection coverage is partial.
41
+ */
42
+ truncated: boolean;
43
+ }
44
+ /**
45
+ * Tokenizer configuration
46
+ */
47
+ export interface TokenizerConfig {
48
+ /** Maximum sequence length */
49
+ maxLength: number;
50
+ /** Whether to lowercase input */
51
+ doLowerCase: boolean;
52
+ }
53
+ export interface TokenizeOptions {
54
+ /** Whether to truncate to the configured maximum sequence length. */
55
+ truncate?: boolean;
56
+ /**
57
+ * Stop after producing this many tokens (including the CLS/SEP special
58
+ * tokens). Bounds memory and CPU on very large inputs: the caller only
59
+ * needs enough tokens to build its windows, so there is no point
60
+ * materializing a multi-million-token array for a multi-MB value.
61
+ * Takes precedence over `truncate` when set.
62
+ */
63
+ maxTokens?: number;
64
+ }
65
+ /**
66
+ * Default tokenizer configuration
67
+ */
68
+ export declare const DEFAULT_TOKENIZER_CONFIG: TokenizerConfig;
69
+ /**
70
+ * WordPiece Tokenizer - supports both HuggingFace JSON and vocab.txt formats
71
+ * Uses a Trie for O(token_length) lookups instead of O(vocab_size)
72
+ */
73
+ export declare class WordPieceTokenizer {
74
+ private vocab;
75
+ private inverseVocab;
76
+ private config;
77
+ private trie;
78
+ private clsId;
79
+ private sepId;
80
+ private padId;
81
+ private unkId;
82
+ private clsToken;
83
+ private sepToken;
84
+ private padToken;
85
+ private unkToken;
86
+ constructor(vocab: Map<string, number>, config?: Partial<TokenizerConfig>);
87
+ /**
88
+ * Insert a token into the trie
89
+ */
90
+ private insertIntoTrie;
91
+ /**
92
+ * Detect special tokens from vocabulary
93
+ */
94
+ private detectSpecialTokens;
95
+ /**
96
+ * Tokenizes text into tokens with offset tracking
97
+ */
98
+ tokenize(text: string, options?: TokenizeOptions): TokenizationResult;
99
+ /**
100
+ * Find the best matching token using trie-based greedy longest-match
101
+ * O(max_token_length) instead of O(vocab_size)
102
+ */
103
+ private findBestToken;
104
+ /**
105
+ * Find the longest matching token in the trie
106
+ */
107
+ private findLongestTrieMatch;
108
+ /**
109
+ * Decodes token IDs back to text
110
+ */
111
+ decode(tokenIds: number[]): string;
112
+ /**
113
+ * Gets vocabulary size
114
+ */
115
+ get vocabSize(): number;
116
+ /**
117
+ * Gets a token ID by string
118
+ */
119
+ getTokenId(token: string): number | undefined;
120
+ /**
121
+ * Gets a token string by ID
122
+ */
123
+ getToken(id: number): string | undefined;
124
+ }
125
+ /**
126
+ * Loads vocabulary from a file (supports tokenizer.json and vocab.txt)
127
+ * Uses storage abstraction for browser compatibility
128
+ */
129
+ export declare function loadVocabFromFile(filePath: string): Promise<Map<string, number>>;
130
+ /**
131
+ * Parses HuggingFace tokenizer.json format
132
+ */
133
+ export declare function parseHFTokenizerJson(content: string): Map<string, number>;
134
+ /**
135
+ * Parses vocabulary from string content (vocab.txt format)
136
+ */
137
+ export declare function parseVocab(content: string): Map<string, number>;
138
+ /**
139
+ * Creates a minimal vocabulary for testing
140
+ */
141
+ export declare function createTestVocab(): Map<string, number>;
142
+ //# sourceMappingURL=tokenizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,mBAAmB;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,cAAc,EAAE,OAAO,CAAC;IACxB,sCAAsC;IACtC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,sBAAsB;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8DAA8D;IAC9D,eAAe,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAChD;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;CACpB;AAsBD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAkBF;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,IAAI,CAAW;IAGvB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAsB7E;;OAEG;IACH,OAAO,CAAC,cAAc;IAYtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,kBAAkB;IAqFzE;;;OAGG;IACH,OAAO,CAAC,aAAa;IAyBrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsC5B;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAmBlC;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;CAGzC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAWtF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAmCzE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAY/D;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrD"}