opencode-rehydra-core 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/LICENSE +22 -0
  2. package/README.en.md +191 -0
  3. package/README.md +191 -0
  4. package/dist/browser.d.ts +54 -0
  5. package/dist/browser.d.ts.map +1 -0
  6. package/dist/browser.js +74 -0
  7. package/dist/browser.js.map +1 -0
  8. package/dist/core/anonymizer.d.ts +309 -0
  9. package/dist/core/anonymizer.d.ts.map +1 -0
  10. package/dist/core/anonymizer.js +555 -0
  11. package/dist/core/anonymizer.js.map +1 -0
  12. package/dist/core/index.d.ts +6 -0
  13. package/dist/core/index.d.ts.map +1 -0
  14. package/dist/core/index.js +6 -0
  15. package/dist/core/index.js.map +1 -0
  16. package/dist/crypto/index.d.ts +6 -0
  17. package/dist/crypto/index.d.ts.map +1 -0
  18. package/dist/crypto/index.js +6 -0
  19. package/dist/crypto/index.js.map +1 -0
  20. package/dist/crypto/pii-map-crypto.d.ts +114 -0
  21. package/dist/crypto/pii-map-crypto.d.ts.map +1 -0
  22. package/dist/crypto/pii-map-crypto.js +228 -0
  23. package/dist/crypto/pii-map-crypto.js.map +1 -0
  24. package/dist/index.d.ts +53 -0
  25. package/dist/index.d.ts.map +1 -0
  26. package/dist/index.js +72 -0
  27. package/dist/index.js.map +1 -0
  28. package/dist/ner/bio-decoder.d.ts +64 -0
  29. package/dist/ner/bio-decoder.d.ts.map +1 -0
  30. package/dist/ner/bio-decoder.js +218 -0
  31. package/dist/ner/bio-decoder.js.map +1 -0
  32. package/dist/ner/index.d.ts +12 -0
  33. package/dist/ner/index.d.ts.map +1 -0
  34. package/dist/ner/index.js +12 -0
  35. package/dist/ner/index.js.map +1 -0
  36. package/dist/ner/inference-server-client.d.ts +81 -0
  37. package/dist/ner/inference-server-client.d.ts.map +1 -0
  38. package/dist/ner/inference-server-client.js +120 -0
  39. package/dist/ner/inference-server-client.js.map +1 -0
  40. package/dist/ner/inference-server-ner-model.d.ts +43 -0
  41. package/dist/ner/inference-server-ner-model.d.ts.map +1 -0
  42. package/dist/ner/inference-server-ner-model.js +119 -0
  43. package/dist/ner/inference-server-ner-model.js.map +1 -0
  44. package/dist/ner/model-manager.d.ts +111 -0
  45. package/dist/ner/model-manager.d.ts.map +1 -0
  46. package/dist/ner/model-manager.js +334 -0
  47. package/dist/ner/model-manager.js.map +1 -0
  48. package/dist/ner/ner-model.d.ts +172 -0
  49. package/dist/ner/ner-model.d.ts.map +1 -0
  50. package/dist/ner/ner-model.js +514 -0
  51. package/dist/ner/ner-model.js.map +1 -0
  52. package/dist/ner/onnx-runtime.browser.d.ts +46 -0
  53. package/dist/ner/onnx-runtime.browser.d.ts.map +1 -0
  54. package/dist/ner/onnx-runtime.browser.js +90 -0
  55. package/dist/ner/onnx-runtime.browser.js.map +1 -0
  56. package/dist/ner/onnx-runtime.d.ts +72 -0
  57. package/dist/ner/onnx-runtime.d.ts.map +1 -0
  58. package/dist/ner/onnx-runtime.js +132 -0
  59. package/dist/ner/onnx-runtime.js.map +1 -0
  60. package/dist/ner/token-window.d.ts +19 -0
  61. package/dist/ner/token-window.d.ts.map +1 -0
  62. package/dist/ner/token-window.js +101 -0
  63. package/dist/ner/token-window.js.map +1 -0
  64. package/dist/ner/tokenizer.d.ts +142 -0
  65. package/dist/ner/tokenizer.d.ts.map +1 -0
  66. package/dist/ner/tokenizer.js +362 -0
  67. package/dist/ner/tokenizer.js.map +1 -0
  68. package/dist/opencode-plugin/git-identity.d.ts +3 -0
  69. package/dist/opencode-plugin/git-identity.d.ts.map +1 -0
  70. package/dist/opencode-plugin/git-identity.js +37 -0
  71. package/dist/opencode-plugin/git-identity.js.map +1 -0
  72. package/dist/opencode-plugin/github-username.d.ts +3 -0
  73. package/dist/opencode-plugin/github-username.d.ts.map +1 -0
  74. package/dist/opencode-plugin/github-username.js +41 -0
  75. package/dist/opencode-plugin/github-username.js.map +1 -0
  76. package/dist/opencode-plugin/identity-recognizer.d.ts +5 -0
  77. package/dist/opencode-plugin/identity-recognizer.d.ts.map +1 -0
  78. package/dist/opencode-plugin/identity-recognizer.js +21 -0
  79. package/dist/opencode-plugin/identity-recognizer.js.map +1 -0
  80. package/dist/opencode-plugin/index.d.ts +7 -0
  81. package/dist/opencode-plugin/index.d.ts.map +1 -0
  82. package/dist/opencode-plugin/index.js +6 -0
  83. package/dist/opencode-plugin/index.js.map +1 -0
  84. package/dist/opencode-plugin/plugin.d.ts +39 -0
  85. package/dist/opencode-plugin/plugin.d.ts.map +1 -0
  86. package/dist/opencode-plugin/plugin.js +294 -0
  87. package/dist/opencode-plugin/plugin.js.map +1 -0
  88. package/dist/opencode-plugin/types.d.ts +33 -0
  89. package/dist/opencode-plugin/types.d.ts.map +1 -0
  90. package/dist/opencode-plugin/types.js +5 -0
  91. package/dist/opencode-plugin/types.js.map +1 -0
  92. package/dist/opencode-plugin/v2/anon-state.d.ts +12 -0
  93. package/dist/opencode-plugin/v2/anon-state.d.ts.map +1 -0
  94. package/dist/opencode-plugin/v2/anon-state.js +14 -0
  95. package/dist/opencode-plugin/v2/anon-state.js.map +1 -0
  96. package/dist/opencode-plugin/v2/deep-rehydrate.d.ts +13 -0
  97. package/dist/opencode-plugin/v2/deep-rehydrate.d.ts.map +1 -0
  98. package/dist/opencode-plugin/v2/deep-rehydrate.js +33 -0
  99. package/dist/opencode-plugin/v2/deep-rehydrate.js.map +1 -0
  100. package/dist/opencode-plugin/v2/index.d.ts +17 -0
  101. package/dist/opencode-plugin/v2/index.d.ts.map +1 -0
  102. package/dist/opencode-plugin/v2/index.js +17 -0
  103. package/dist/opencode-plugin/v2/index.js.map +1 -0
  104. package/dist/opencode-plugin/v2/logger.d.ts +24 -0
  105. package/dist/opencode-plugin/v2/logger.d.ts.map +1 -0
  106. package/dist/opencode-plugin/v2/logger.js +73 -0
  107. package/dist/opencode-plugin/v2/logger.js.map +1 -0
  108. package/dist/opencode-plugin/v2/options.d.ts +40 -0
  109. package/dist/opencode-plugin/v2/options.d.ts.map +1 -0
  110. package/dist/opencode-plugin/v2/options.js +132 -0
  111. package/dist/opencode-plugin/v2/options.js.map +1 -0
  112. package/dist/opencode-plugin/v2/register-hooks.d.ts +43 -0
  113. package/dist/opencode-plugin/v2/register-hooks.d.ts.map +1 -0
  114. package/dist/opencode-plugin/v2/register-hooks.js +160 -0
  115. package/dist/opencode-plugin/v2/register-hooks.js.map +1 -0
  116. package/dist/opencode-plugin/v2/rehydrate-tail.d.ts +101 -0
  117. package/dist/opencode-plugin/v2/rehydrate-tail.d.ts.map +1 -0
  118. package/dist/opencode-plugin/v2/rehydrate-tail.js +147 -0
  119. package/dist/opencode-plugin/v2/rehydrate-tail.js.map +1 -0
  120. package/dist/opencode-plugin/v2/response-rehydrate.d.ts +78 -0
  121. package/dist/opencode-plugin/v2/response-rehydrate.d.ts.map +1 -0
  122. package/dist/opencode-plugin/v2/response-rehydrate.js +210 -0
  123. package/dist/opencode-plugin/v2/response-rehydrate.js.map +1 -0
  124. package/dist/opencode-plugin/v2/scrub-messages.d.ts +39 -0
  125. package/dist/opencode-plugin/v2/scrub-messages.d.ts.map +1 -0
  126. package/dist/opencode-plugin/v2/scrub-messages.js +260 -0
  127. package/dist/opencode-plugin/v2/scrub-messages.js.map +1 -0
  128. package/dist/opencode-plugin/v2/sessions.d.ts +17 -0
  129. package/dist/opencode-plugin/v2/sessions.d.ts.map +1 -0
  130. package/dist/opencode-plugin/v2/sessions.js +23 -0
  131. package/dist/opencode-plugin/v2/sessions.js.map +1 -0
  132. package/dist/opencode-plugin/v2/sse.d.ts +101 -0
  133. package/dist/opencode-plugin/v2/sse.d.ts.map +1 -0
  134. package/dist/opencode-plugin/v2/sse.js +235 -0
  135. package/dist/opencode-plugin/v2/sse.js.map +1 -0
  136. package/dist/opencode-plugin/v2/types.d.ts +188 -0
  137. package/dist/opencode-plugin/v2/types.d.ts.map +1 -0
  138. package/dist/opencode-plugin/v2/types.js +10 -0
  139. package/dist/opencode-plugin/v2/types.js.map +1 -0
  140. package/dist/opencode-plugin/vcs-command.d.ts +3 -0
  141. package/dist/opencode-plugin/vcs-command.d.ts.map +1 -0
  142. package/dist/opencode-plugin/vcs-command.js +34 -0
  143. package/dist/opencode-plugin/vcs-command.js.map +1 -0
  144. package/dist/pipeline/index.d.ts +12 -0
  145. package/dist/pipeline/index.d.ts.map +1 -0
  146. package/dist/pipeline/index.js +12 -0
  147. package/dist/pipeline/index.js.map +1 -0
  148. package/dist/pipeline/prenormalize.d.ts +48 -0
  149. package/dist/pipeline/prenormalize.d.ts.map +1 -0
  150. package/dist/pipeline/prenormalize.js +94 -0
  151. package/dist/pipeline/prenormalize.js.map +1 -0
  152. package/dist/pipeline/resolver.d.ts +56 -0
  153. package/dist/pipeline/resolver.d.ts.map +1 -0
  154. package/dist/pipeline/resolver.js +239 -0
  155. package/dist/pipeline/resolver.js.map +1 -0
  156. package/dist/pipeline/semantic-data-loader.d.ts +171 -0
  157. package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
  158. package/dist/pipeline/semantic-data-loader.js +677 -0
  159. package/dist/pipeline/semantic-data-loader.js.map +1 -0
  160. package/dist/pipeline/semantic-enricher.d.ts +112 -0
  161. package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
  162. package/dist/pipeline/semantic-enricher.js +322 -0
  163. package/dist/pipeline/semantic-enricher.js.map +1 -0
  164. package/dist/pipeline/tagger.d.ts +121 -0
  165. package/dist/pipeline/tagger.d.ts.map +1 -0
  166. package/dist/pipeline/tagger.js +545 -0
  167. package/dist/pipeline/tagger.js.map +1 -0
  168. package/dist/pipeline/title-extractor.d.ts +79 -0
  169. package/dist/pipeline/title-extractor.d.ts.map +1 -0
  170. package/dist/pipeline/title-extractor.js +801 -0
  171. package/dist/pipeline/title-extractor.js.map +1 -0
  172. package/dist/pipeline/validator.d.ts +65 -0
  173. package/dist/pipeline/validator.d.ts.map +1 -0
  174. package/dist/pipeline/validator.js +292 -0
  175. package/dist/pipeline/validator.js.map +1 -0
  176. package/dist/proxy/index.d.ts +13 -0
  177. package/dist/proxy/index.d.ts.map +1 -0
  178. package/dist/proxy/index.js +12 -0
  179. package/dist/proxy/index.js.map +1 -0
  180. package/dist/proxy/providers/anthropic.d.ts +28 -0
  181. package/dist/proxy/providers/anthropic.d.ts.map +1 -0
  182. package/dist/proxy/providers/anthropic.js +265 -0
  183. package/dist/proxy/providers/anthropic.js.map +1 -0
  184. package/dist/proxy/providers/index.d.ts +19 -0
  185. package/dist/proxy/providers/index.d.ts.map +1 -0
  186. package/dist/proxy/providers/index.js +41 -0
  187. package/dist/proxy/providers/index.js.map +1 -0
  188. package/dist/proxy/providers/openai.d.ts +27 -0
  189. package/dist/proxy/providers/openai.d.ts.map +1 -0
  190. package/dist/proxy/providers/openai.js +203 -0
  191. package/dist/proxy/providers/openai.js.map +1 -0
  192. package/dist/proxy/providers/streaming-tool-loop.d.ts +12 -0
  193. package/dist/proxy/providers/streaming-tool-loop.d.ts.map +1 -0
  194. package/dist/proxy/providers/streaming-tool-loop.js +223 -0
  195. package/dist/proxy/providers/streaming-tool-loop.js.map +1 -0
  196. package/dist/proxy/providers/types.d.ts +109 -0
  197. package/dist/proxy/providers/types.d.ts.map +1 -0
  198. package/dist/proxy/providers/types.js +2 -0
  199. package/dist/proxy/providers/types.js.map +1 -0
  200. package/dist/proxy/proxy-server.d.ts +61 -0
  201. package/dist/proxy/proxy-server.d.ts.map +1 -0
  202. package/dist/proxy/proxy-server.js +146 -0
  203. package/dist/proxy/proxy-server.js.map +1 -0
  204. package/dist/proxy/rehydra-fetch.d.ts +35 -0
  205. package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
  206. package/dist/proxy/rehydra-fetch.js +631 -0
  207. package/dist/proxy/rehydra-fetch.js.map +1 -0
  208. package/dist/proxy/rehydra-proxy.d.ts +40 -0
  209. package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
  210. package/dist/proxy/rehydra-proxy.js +150 -0
  211. package/dist/proxy/rehydra-proxy.js.map +1 -0
  212. package/dist/proxy/sse-parser.d.ts +59 -0
  213. package/dist/proxy/sse-parser.d.ts.map +1 -0
  214. package/dist/proxy/sse-parser.js +112 -0
  215. package/dist/proxy/sse-parser.js.map +1 -0
  216. package/dist/proxy/system-instruction.d.ts +19 -0
  217. package/dist/proxy/system-instruction.d.ts.map +1 -0
  218. package/dist/proxy/system-instruction.js +33 -0
  219. package/dist/proxy/system-instruction.js.map +1 -0
  220. package/dist/proxy/tool-loop-stream.d.ts +7 -0
  221. package/dist/proxy/tool-loop-stream.d.ts.map +1 -0
  222. package/dist/proxy/tool-loop-stream.js +65 -0
  223. package/dist/proxy/tool-loop-stream.js.map +1 -0
  224. package/dist/proxy/types.d.ts +138 -0
  225. package/dist/proxy/types.d.ts.map +1 -0
  226. package/dist/proxy/types.js +5 -0
  227. package/dist/proxy/types.js.map +1 -0
  228. package/dist/proxy/wrap-client.d.ts +47 -0
  229. package/dist/proxy/wrap-client.d.ts.map +1 -0
  230. package/dist/proxy/wrap-client.js +70 -0
  231. package/dist/proxy/wrap-client.js.map +1 -0
  232. package/dist/recognizers/address.d.ts +2 -0
  233. package/dist/recognizers/address.d.ts.map +1 -0
  234. package/dist/recognizers/address.js +23 -0
  235. package/dist/recognizers/address.js.map +1 -0
  236. package/dist/recognizers/base.d.ts +78 -0
  237. package/dist/recognizers/base.d.ts.map +1 -0
  238. package/dist/recognizers/base.js +100 -0
  239. package/dist/recognizers/base.js.map +1 -0
  240. package/dist/recognizers/bic-swift.d.ts +10 -0
  241. package/dist/recognizers/bic-swift.d.ts.map +1 -0
  242. package/dist/recognizers/bic-swift.js +107 -0
  243. package/dist/recognizers/bic-swift.js.map +1 -0
  244. package/dist/recognizers/credit-card.d.ts +32 -0
  245. package/dist/recognizers/credit-card.d.ts.map +1 -0
  246. package/dist/recognizers/credit-card.js +160 -0
  247. package/dist/recognizers/credit-card.js.map +1 -0
  248. package/dist/recognizers/custom-id.d.ts +28 -0
  249. package/dist/recognizers/custom-id.d.ts.map +1 -0
  250. package/dist/recognizers/custom-id.js +116 -0
  251. package/dist/recognizers/custom-id.js.map +1 -0
  252. package/dist/recognizers/date.d.ts +10 -0
  253. package/dist/recognizers/date.d.ts.map +1 -0
  254. package/dist/recognizers/date.js +228 -0
  255. package/dist/recognizers/date.js.map +1 -0
  256. package/dist/recognizers/email.d.ts +10 -0
  257. package/dist/recognizers/email.d.ts.map +1 -0
  258. package/dist/recognizers/email.js +75 -0
  259. package/dist/recognizers/email.js.map +1 -0
  260. package/dist/recognizers/iban.d.ts +14 -0
  261. package/dist/recognizers/iban.d.ts.map +1 -0
  262. package/dist/recognizers/iban.js +67 -0
  263. package/dist/recognizers/iban.js.map +1 -0
  264. package/dist/recognizers/index.d.ts +25 -0
  265. package/dist/recognizers/index.d.ts.map +1 -0
  266. package/dist/recognizers/index.js +53 -0
  267. package/dist/recognizers/index.js.map +1 -0
  268. package/dist/recognizers/ip-address.d.ts +14 -0
  269. package/dist/recognizers/ip-address.d.ts.map +1 -0
  270. package/dist/recognizers/ip-address.js +183 -0
  271. package/dist/recognizers/ip-address.js.map +1 -0
  272. package/dist/recognizers/phone.d.ts +10 -0
  273. package/dist/recognizers/phone.d.ts.map +1 -0
  274. package/dist/recognizers/phone.js +145 -0
  275. package/dist/recognizers/phone.js.map +1 -0
  276. package/dist/recognizers/postal-code.d.ts +16 -0
  277. package/dist/recognizers/postal-code.d.ts.map +1 -0
  278. package/dist/recognizers/postal-code.js +42 -0
  279. package/dist/recognizers/postal-code.js.map +1 -0
  280. package/dist/recognizers/registry.d.ts +59 -0
  281. package/dist/recognizers/registry.d.ts.map +1 -0
  282. package/dist/recognizers/registry.js +113 -0
  283. package/dist/recognizers/registry.js.map +1 -0
  284. package/dist/recognizers/secrets/api-key.d.ts +6 -0
  285. package/dist/recognizers/secrets/api-key.d.ts.map +1 -0
  286. package/dist/recognizers/secrets/api-key.js +40 -0
  287. package/dist/recognizers/secrets/api-key.js.map +1 -0
  288. package/dist/recognizers/secrets/aws-credentials.d.ts +7 -0
  289. package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -0
  290. package/dist/recognizers/secrets/aws-credentials.js +68 -0
  291. package/dist/recognizers/secrets/aws-credentials.js.map +1 -0
  292. package/dist/recognizers/secrets/config-secret.d.ts +8 -0
  293. package/dist/recognizers/secrets/config-secret.d.ts.map +1 -0
  294. package/dist/recognizers/secrets/config-secret.js +59 -0
  295. package/dist/recognizers/secrets/config-secret.js.map +1 -0
  296. package/dist/recognizers/secrets/connection-string.d.ts +6 -0
  297. package/dist/recognizers/secrets/connection-string.d.ts.map +1 -0
  298. package/dist/recognizers/secrets/connection-string.js +40 -0
  299. package/dist/recognizers/secrets/connection-string.js.map +1 -0
  300. package/dist/recognizers/secrets/env-var.d.ts +8 -0
  301. package/dist/recognizers/secrets/env-var.d.ts.map +1 -0
  302. package/dist/recognizers/secrets/env-var.js +54 -0
  303. package/dist/recognizers/secrets/env-var.js.map +1 -0
  304. package/dist/recognizers/secrets/index.d.ts +22 -0
  305. package/dist/recognizers/secrets/index.d.ts.map +1 -0
  306. package/dist/recognizers/secrets/index.js +41 -0
  307. package/dist/recognizers/secrets/index.js.map +1 -0
  308. package/dist/recognizers/secrets/jwt.d.ts +6 -0
  309. package/dist/recognizers/secrets/jwt.d.ts.map +1 -0
  310. package/dist/recognizers/secrets/jwt.js +47 -0
  311. package/dist/recognizers/secrets/jwt.js.map +1 -0
  312. package/dist/recognizers/secrets/key-patterns.d.ts +10 -0
  313. package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -0
  314. package/dist/recognizers/secrets/key-patterns.js +15 -0
  315. package/dist/recognizers/secrets/key-patterns.js.map +1 -0
  316. package/dist/recognizers/secrets/literal-value.d.ts +10 -0
  317. package/dist/recognizers/secrets/literal-value.d.ts.map +1 -0
  318. package/dist/recognizers/secrets/literal-value.js +48 -0
  319. package/dist/recognizers/secrets/literal-value.js.map +1 -0
  320. package/dist/recognizers/secrets/private-key.d.ts +6 -0
  321. package/dist/recognizers/secrets/private-key.d.ts.map +1 -0
  322. package/dist/recognizers/secrets/private-key.js +16 -0
  323. package/dist/recognizers/secrets/private-key.js.map +1 -0
  324. package/dist/recognizers/url.d.ts +14 -0
  325. package/dist/recognizers/url.d.ts.map +1 -0
  326. package/dist/recognizers/url.js +121 -0
  327. package/dist/recognizers/url.js.map +1 -0
  328. package/dist/storage/browser.d.ts +13 -0
  329. package/dist/storage/browser.d.ts.map +1 -0
  330. package/dist/storage/browser.js +13 -0
  331. package/dist/storage/browser.js.map +1 -0
  332. package/dist/storage/in-memory.d.ts +60 -0
  333. package/dist/storage/in-memory.d.ts.map +1 -0
  334. package/dist/storage/in-memory.js +108 -0
  335. package/dist/storage/in-memory.js.map +1 -0
  336. package/dist/storage/index.d.ts +11 -0
  337. package/dist/storage/index.d.ts.map +1 -0
  338. package/dist/storage/index.js +11 -0
  339. package/dist/storage/index.js.map +1 -0
  340. package/dist/storage/indexeddb.d.ts +69 -0
  341. package/dist/storage/indexeddb.d.ts.map +1 -0
  342. package/dist/storage/indexeddb.js +274 -0
  343. package/dist/storage/indexeddb.js.map +1 -0
  344. package/dist/storage/session-base.d.ts +34 -0
  345. package/dist/storage/session-base.d.ts.map +1 -0
  346. package/dist/storage/session-base.js +122 -0
  347. package/dist/storage/session-base.js.map +1 -0
  348. package/dist/storage/session.d.ts +19 -0
  349. package/dist/storage/session.d.ts.map +1 -0
  350. package/dist/storage/session.js +31 -0
  351. package/dist/storage/session.js.map +1 -0
  352. package/dist/storage/sqlite.d.ts +80 -0
  353. package/dist/storage/sqlite.d.ts.map +1 -0
  354. package/dist/storage/sqlite.js +255 -0
  355. package/dist/storage/sqlite.js.map +1 -0
  356. package/dist/storage/types.d.ts +208 -0
  357. package/dist/storage/types.d.ts.map +1 -0
  358. package/dist/storage/types.js +6 -0
  359. package/dist/storage/types.js.map +1 -0
  360. package/dist/streaming/anonymizer-stream.d.ts +63 -0
  361. package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
  362. package/dist/streaming/anonymizer-stream.js +184 -0
  363. package/dist/streaming/anonymizer-stream.js.map +1 -0
  364. package/dist/streaming/index.d.ts +9 -0
  365. package/dist/streaming/index.d.ts.map +1 -0
  366. package/dist/streaming/index.js +8 -0
  367. package/dist/streaming/index.js.map +1 -0
  368. package/dist/streaming/sentence-buffer.d.ts +78 -0
  369. package/dist/streaming/sentence-buffer.d.ts.map +1 -0
  370. package/dist/streaming/sentence-buffer.js +239 -0
  371. package/dist/streaming/sentence-buffer.js.map +1 -0
  372. package/dist/streaming/stream-factory.d.ts +38 -0
  373. package/dist/streaming/stream-factory.d.ts.map +1 -0
  374. package/dist/streaming/stream-factory.js +69 -0
  375. package/dist/streaming/stream-factory.js.map +1 -0
  376. package/dist/streaming/types.d.ts +121 -0
  377. package/dist/streaming/types.d.ts.map +1 -0
  378. package/dist/streaming/types.js +5 -0
  379. package/dist/streaming/types.js.map +1 -0
  380. package/dist/types/index.d.ts +269 -0
  381. package/dist/types/index.d.ts.map +1 -0
  382. package/dist/types/index.js +95 -0
  383. package/dist/types/index.js.map +1 -0
  384. package/dist/types/pii-types.d.ts +64 -0
  385. package/dist/types/pii-types.d.ts.map +1 -0
  386. package/dist/types/pii-types.js +153 -0
  387. package/dist/types/pii-types.js.map +1 -0
  388. package/dist/utils/iban-checksum.d.ts +23 -0
  389. package/dist/utils/iban-checksum.d.ts.map +1 -0
  390. package/dist/utils/iban-checksum.js +106 -0
  391. package/dist/utils/iban-checksum.js.map +1 -0
  392. package/dist/utils/index.d.ts +11 -0
  393. package/dist/utils/index.d.ts.map +1 -0
  394. package/dist/utils/index.js +11 -0
  395. package/dist/utils/index.js.map +1 -0
  396. package/dist/utils/json-walk.d.ts +21 -0
  397. package/dist/utils/json-walk.d.ts.map +1 -0
  398. package/dist/utils/json-walk.js +44 -0
  399. package/dist/utils/json-walk.js.map +1 -0
  400. package/dist/utils/luhn.d.ts +17 -0
  401. package/dist/utils/luhn.d.ts.map +1 -0
  402. package/dist/utils/luhn.js +55 -0
  403. package/dist/utils/luhn.js.map +1 -0
  404. package/dist/utils/offsets.d.ts +86 -0
  405. package/dist/utils/offsets.d.ts.map +1 -0
  406. package/dist/utils/offsets.js +124 -0
  407. package/dist/utils/offsets.js.map +1 -0
  408. package/dist/utils/path.d.ts +34 -0
  409. package/dist/utils/path.d.ts.map +1 -0
  410. package/dist/utils/path.js +96 -0
  411. package/dist/utils/path.js.map +1 -0
  412. package/dist/utils/regex.d.ts +11 -0
  413. package/dist/utils/regex.d.ts.map +1 -0
  414. package/dist/utils/regex.js +14 -0
  415. package/dist/utils/regex.js.map +1 -0
  416. package/dist/utils/storage-browser.d.ts +51 -0
  417. package/dist/utils/storage-browser.d.ts.map +1 -0
  418. package/dist/utils/storage-browser.js +381 -0
  419. package/dist/utils/storage-browser.js.map +1 -0
  420. package/dist/utils/storage-node.d.ts +51 -0
  421. package/dist/utils/storage-node.d.ts.map +1 -0
  422. package/dist/utils/storage-node.js +153 -0
  423. package/dist/utils/storage-node.js.map +1 -0
  424. package/dist/utils/storage.browser.d.ts +75 -0
  425. package/dist/utils/storage.browser.d.ts.map +1 -0
  426. package/dist/utils/storage.browser.js +69 -0
  427. package/dist/utils/storage.browser.js.map +1 -0
  428. package/dist/utils/storage.d.ts +73 -0
  429. package/dist/utils/storage.d.ts.map +1 -0
  430. package/dist/utils/storage.js +69 -0
  431. package/dist/utils/storage.js.map +1 -0
  432. package/package.json +148 -0
@@ -0,0 +1,111 @@
1
+ /**
2
+ * NER Model Manager
3
+ * Handles automatic downloading and caching of NER models from Hugging Face Hub
4
+ * Browser-compatible using storage abstraction layer
5
+ */
6
+ /**
7
+ * Available NER model variants
8
+ */
9
+ export type NERModelMode = "standard" | "quantized" | "disabled" | "custom";
10
+ /**
11
+ * Model file info
12
+ */
13
+ export interface ModelFileInfo {
14
+ /** Filename in the repo */
15
+ repoFile: string;
16
+ /** Local filename */
17
+ localFile: string;
18
+ /** Whether file is required */
19
+ required: boolean;
20
+ }
21
+ /**
22
+ * Model registry entry
23
+ */
24
+ export interface ModelInfo {
25
+ /** Model identifier */
26
+ id: string;
27
+ /** Human-readable name */
28
+ name: string;
29
+ /** Description */
30
+ description: string;
31
+ /** Approximate size */
32
+ size: string;
33
+ /** Hugging Face repo ID */
34
+ hfRepo: string;
35
+ /** Subfolder in repo (for models with multiple variants) */
36
+ hfSubfolder?: string;
37
+ /** Files to download */
38
+ files: ModelFileInfo[];
39
+ /** Label map for this model */
40
+ labelMap: string[];
41
+ }
42
+ /**
43
+ * Registry of available models hosted on Hugging Face Hub
44
+ *
45
+ * Using ELAN's ONNX exports which are optimized for JS/ONNX runtime
46
+ * https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx
47
+ */
48
+ export declare const MODEL_REGISTRY: Record<"standard" | "quantized", ModelInfo>;
49
+ /**
50
+ * Gets the cache directory for models
51
+ * Uses platform-specific cache location (or virtual path in browser)
52
+ */
53
+ export declare function getModelCacheDir(): Promise<string>;
54
+ /**
55
+ * Gets the path to a specific model variant
56
+ */
57
+ export declare function getModelPath(mode: "standard" | "quantized"): Promise<string>;
58
+ /**
59
+ * Checks if a model is already downloaded
60
+ */
61
+ export declare function isModelDownloaded(mode: "standard" | "quantized"): Promise<boolean>;
62
+ /**
63
+ * Progress callback for downloads
64
+ */
65
+ export type DownloadProgressCallback = (progress: {
66
+ file: string;
67
+ bytesDownloaded: number;
68
+ totalBytes: number | null;
69
+ percent: number | null;
70
+ }) => void;
71
+ /**
72
+ * Downloads a model variant from Hugging Face Hub
73
+ */
74
+ export declare function downloadModel(mode: "standard" | "quantized", onProgress?: DownloadProgressCallback, onStatus?: (status: string) => void): Promise<string>;
75
+ /**
76
+ * Gets model paths if available, or downloads if needed
77
+ */
78
+ export declare function ensureModel(mode: "standard" | "quantized", options?: {
79
+ autoDownload?: boolean;
80
+ onProgress?: DownloadProgressCallback;
81
+ onStatus?: (status: string) => void;
82
+ }): Promise<{
83
+ modelPath: string;
84
+ vocabPath: string;
85
+ labelMapPath: string;
86
+ }>;
87
+ /**
88
+ * Loads the label map for a registry model, preferring the label_map.json
89
+ * persisted by ensureModel and falling back to the registry's map for the
90
+ * mode. The registry is authoritative — it is the source ensureModel writes
91
+ * to the file — so a missing or corrupted file never changes the decoded
92
+ * label order (a mismatched map silently misclassifies every NER entity).
93
+ */
94
+ export declare function loadLabelMap(mode: "standard" | "quantized", labelMapPath: string): Promise<string[]>;
95
+ /**
96
+ * Clears cached models
97
+ */
98
+ export declare function clearModelCache(mode?: "standard" | "quantized"): Promise<void>;
99
+ /**
100
+ * Lists downloaded models
101
+ */
102
+ export declare function listDownloadedModels(): Promise<Array<{
103
+ mode: "standard" | "quantized";
104
+ path: string;
105
+ size: string;
106
+ }>>;
107
+ /**
108
+ * Gets info about available models
109
+ */
110
+ export declare function getModelInfo(mode: "standard" | "quantized"): ModelInfo;
111
+ //# sourceMappingURL=model-manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,UAAU,GAAG,WAAW,EAAE,SAAS,CAiDtE,CAAC;AAiCF;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAGxD;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,OAAO,CAAC,OAAO,CAAC,CAyBlB;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,KAAK,IAAI,CAAC;AAsGX;;GAEG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,UAAU,CAAC,EAAE,wBAAwB,EACrC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC,CAsDjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,wBAAwB,CAAC;IACtC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAiCzE;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAoBnB;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,GAC9B,OAAO,CAAC,IAAI,CAAC,CAUf;AAED;;GAEG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CACnD,KAAK,CAAC;IAAE,IAAI,EAAE,UAAU,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CACtE,CAgBA;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAEtE"}
@@ -0,0 +1,334 @@
1
+ /**
2
+ * NER Model Manager
3
+ * Handles automatic downloading and caching of NER models from Hugging Face Hub
4
+ * Browser-compatible using storage abstraction layer
5
+ */
6
+ import { getStorageProvider } from "#storage";
7
+ import { join, basename } from "../utils/path.js";
8
+ /**
9
+ * Registry of available models hosted on Hugging Face Hub
10
+ *
11
+ * Using ELAN's ONNX exports which are optimized for JS/ONNX runtime
12
+ * https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx
13
+ */
14
+ export const MODEL_REGISTRY = {
15
+ standard: {
16
+ id: "xlm-roberta-ner-standard",
17
+ name: "XLM-RoBERTa NER (Standard)",
18
+ description: "Multilingual NER model supporting EN, DE, FR, ES, and more",
19
+ size: "~1.1 GB",
20
+ hfRepo: "tjruesch/xlm-roberta-base-ner-hrl-onnx",
21
+ hfSubfolder: "onnx",
22
+ files: [
23
+ { repoFile: "model.onnx", localFile: "model.onnx", required: true },
24
+ ],
25
+ labelMap: [
26
+ "O",
27
+ "B-DATE",
28
+ "I-DATE",
29
+ "B-PER",
30
+ "I-PER",
31
+ "B-ORG",
32
+ "I-ORG",
33
+ "B-LOC",
34
+ "I-LOC",
35
+ ],
36
+ },
37
+ quantized: {
38
+ id: "xlm-roberta-ner-quantized",
39
+ name: "XLM-RoBERTa NER (Quantized)",
40
+ description: "Quantized version, ~4x smaller with minimal accuracy loss",
41
+ size: "~265 MB",
42
+ hfRepo: "tjruesch/xlm-roberta-base-ner-hrl-onnx",
43
+ hfSubfolder: "onnx",
44
+ files: [
45
+ {
46
+ repoFile: "model_quantized.onnx",
47
+ localFile: "model.onnx",
48
+ required: true,
49
+ },
50
+ ],
51
+ labelMap: [
52
+ "O",
53
+ "B-DATE",
54
+ "I-DATE",
55
+ "B-PER",
56
+ "I-PER",
57
+ "B-ORG",
58
+ "I-ORG",
59
+ "B-LOC",
60
+ "I-LOC",
61
+ ],
62
+ },
63
+ };
64
+ /**
65
+ * Shared tokenizer files (same for both variants)
66
+ */
67
+ const TOKENIZER_FILES = [
68
+ { repoFile: "tokenizer.json", localFile: "tokenizer.json", required: true },
69
+ {
70
+ repoFile: "tokenizer_config.json",
71
+ localFile: "tokenizer_config.json",
72
+ required: false,
73
+ },
74
+ {
75
+ repoFile: "special_tokens_map.json",
76
+ localFile: "special_tokens_map.json",
77
+ required: false,
78
+ },
79
+ { repoFile: "config.json", localFile: "config.json", required: false },
80
+ ];
81
+ // Cached storage provider
82
+ let storageProvider = null;
83
+ /**
84
+ * Gets the storage provider (lazily initialized)
85
+ */
86
+ async function getStorage() {
87
+ if (storageProvider === null) {
88
+ storageProvider = await getStorageProvider();
89
+ }
90
+ return storageProvider;
91
+ }
92
+ /**
93
+ * Gets the cache directory for models
94
+ * Uses platform-specific cache location (or virtual path in browser)
95
+ */
96
+ export async function getModelCacheDir() {
97
+ const storage = await getStorage();
98
+ return storage.getCacheDir("models");
99
+ }
100
+ /**
101
+ * Gets the path to a specific model variant
102
+ */
103
+ export async function getModelPath(mode) {
104
+ const cacheDir = await getModelCacheDir();
105
+ return join(cacheDir, mode);
106
+ }
107
+ /**
108
+ * Checks if a model is already downloaded
109
+ */
110
+ export async function isModelDownloaded(mode) {
111
+ const storage = await getStorage();
112
+ const modelDir = await getModelPath(mode);
113
+ const info = MODEL_REGISTRY[mode];
114
+ try {
115
+ // Check if model file exists
116
+ const modelFile = info.files.find((f) => f.required && f.localFile.includes("model"));
117
+ if (modelFile) {
118
+ const modelExists = await storage.exists(join(modelDir, modelFile.localFile));
119
+ if (!modelExists)
120
+ return false;
121
+ }
122
+ // Check if tokenizer exists
123
+ const tokenizerExists = await storage.exists(join(modelDir, "tokenizer.json"));
124
+ return tokenizerExists;
125
+ }
126
+ catch {
127
+ return false;
128
+ }
129
+ }
130
+ /**
131
+ * Builds a Hugging Face Hub download URL
132
+ */
133
+ function getHuggingFaceUrl(repo, filename, subfolder) {
134
+ const filePath = subfolder !== undefined && subfolder !== ""
135
+ ? `${subfolder}/${filename}`
136
+ : filename;
137
+ return `https://huggingface.co/${repo}/resolve/main/${filePath}`;
138
+ }
139
+ /**
140
+ * Downloads a file from URL and returns the data
141
+ */
142
+ async function downloadFileData(url, fileName, onProgress) {
143
+ const response = await fetch(url, {
144
+ headers: {
145
+ "User-Agent": "rehydra/1.0.0",
146
+ },
147
+ });
148
+ if (!response.ok) {
149
+ if (response.status === 404) {
150
+ throw new Error(`File not found: ${url}`);
151
+ }
152
+ throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
153
+ }
154
+ const totalBytes = response.headers.get("content-length");
155
+ const total = totalBytes !== null && totalBytes !== "" ? parseInt(totalBytes, 10) : null;
156
+ const reader = response.body?.getReader();
157
+ if (reader === undefined) {
158
+ throw new Error("Response body is not readable");
159
+ }
160
+ const chunks = [];
161
+ let bytesDownloaded = 0;
162
+ // eslint-disable-next-line no-constant-condition
163
+ while (true) {
164
+ const result = await reader.read();
165
+ if (result.done)
166
+ break;
167
+ const value = result.value;
168
+ chunks.push(value);
169
+ bytesDownloaded += value.length;
170
+ if (onProgress) {
171
+ onProgress({
172
+ file: fileName,
173
+ bytesDownloaded,
174
+ totalBytes: total,
175
+ percent: total !== null && total > 0
176
+ ? Math.round((bytesDownloaded / total) * 100)
177
+ : null,
178
+ });
179
+ }
180
+ }
181
+ // Concatenate all chunks
182
+ const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
183
+ const result = new Uint8Array(totalLength);
184
+ let offset = 0;
185
+ for (const chunk of chunks) {
186
+ result.set(chunk, offset);
187
+ offset += chunk.length;
188
+ }
189
+ return result;
190
+ }
191
+ /**
192
+ * Downloads a file from URL to storage
193
+ */
194
+ async function downloadFile(url, destPath, onProgress) {
195
+ const storage = await getStorage();
196
+ const fileName = basename(destPath);
197
+ const data = await downloadFileData(url, fileName, onProgress);
198
+ await storage.writeFile(destPath, data);
199
+ }
200
+ /**
201
+ * Downloads a model variant from Hugging Face Hub
202
+ */
203
+ export async function downloadModel(mode, onProgress, onStatus) {
204
+ const storage = await getStorage();
205
+ const info = MODEL_REGISTRY[mode];
206
+ const modelDir = await getModelPath(mode);
207
+ // Create directory
208
+ await storage.mkdir(modelDir);
209
+ onStatus?.(`Downloading ${info.name} from Hugging Face Hub...`);
210
+ onStatus?.(`Repository: ${info.hfRepo}`);
211
+ // Download model files
212
+ for (const file of info.files) {
213
+ const url = getHuggingFaceUrl(info.hfRepo, file.repoFile, info.hfSubfolder);
214
+ const destPath = join(modelDir, file.localFile);
215
+ onStatus?.(`Downloading ${file.repoFile}...`);
216
+ try {
217
+ await downloadFile(url, destPath, onProgress);
218
+ }
219
+ catch (e) {
220
+ if (file.required) {
221
+ throw new Error(`Failed to download required file ${file.repoFile}: ${String(e)}`);
222
+ }
223
+ // Optional files can fail silently
224
+ onStatus?.(`Skipping optional file ${file.repoFile}`);
225
+ }
226
+ }
227
+ // Download tokenizer files (from repo root, not subfolder)
228
+ for (const file of TOKENIZER_FILES) {
229
+ const url = getHuggingFaceUrl(info.hfRepo, file.repoFile);
230
+ const destPath = join(modelDir, file.localFile);
231
+ try {
232
+ await downloadFile(url, destPath, onProgress);
233
+ }
234
+ catch (e) {
235
+ if (file.required) {
236
+ throw new Error(`Failed to download required file ${file.repoFile}: ${String(e)}`);
237
+ }
238
+ }
239
+ }
240
+ // Write label map
241
+ const labelMapPath = join(modelDir, "label_map.json");
242
+ await storage.writeFile(labelMapPath, JSON.stringify(info.labelMap, null, 2));
243
+ onStatus?.("Download complete!");
244
+ return modelDir;
245
+ }
246
+ /**
247
+ * Gets model paths if available, or downloads if needed
248
+ */
249
+ export async function ensureModel(mode, options = {}) {
250
+ const { autoDownload = true, onProgress, onStatus } = options;
251
+ const modelDir = await getModelPath(mode);
252
+ const info = MODEL_REGISTRY[mode];
253
+ // Check if already downloaded
254
+ const isDownloaded = await isModelDownloaded(mode);
255
+ if (!isDownloaded) {
256
+ if (!autoDownload) {
257
+ throw new Error(`NER model '${mode}' not found at ${modelDir}.\n\n` +
258
+ `To download automatically, use:\n` +
259
+ ` createAnonymizer({ ner: { mode: '${mode}', autoDownload: true } })\n\n` +
260
+ `Or use regex-only mode:\n` +
261
+ ` createAnonymizer({ ner: { mode: 'disabled' } })`);
262
+ }
263
+ await downloadModel(mode, onProgress, onStatus);
264
+ }
265
+ else {
266
+ onStatus?.(`Using cached model: ${info.name}`);
267
+ }
268
+ // Find model file
269
+ const modelFile = info.files.find((f) => f.localFile === "model.onnx");
270
+ return {
271
+ modelPath: join(modelDir, modelFile?.localFile ?? "model.onnx"),
272
+ vocabPath: join(modelDir, "tokenizer.json"),
273
+ labelMapPath: join(modelDir, "label_map.json"),
274
+ };
275
+ }
276
+ /**
277
+ * Loads the label map for a registry model, preferring the label_map.json
278
+ * persisted by ensureModel and falling back to the registry's map for the
279
+ * mode. The registry is authoritative — it is the source ensureModel writes
280
+ * to the file — so a missing or corrupted file never changes the decoded
281
+ * label order (a mismatched map silently misclassifies every NER entity).
282
+ */
283
+ export async function loadLabelMap(mode, labelMapPath) {
284
+ const registryLabelMap = MODEL_REGISTRY[mode].labelMap;
285
+ try {
286
+ const storage = await getStorage();
287
+ const content = await storage.readTextFile(labelMapPath);
288
+ const parsed = JSON.parse(content);
289
+ if (Array.isArray(parsed) &&
290
+ parsed.length > 0 &&
291
+ parsed.every((label) => typeof label === "string")) {
292
+ return parsed;
293
+ }
294
+ }
295
+ catch {
296
+ // Fall through to registry map
297
+ }
298
+ return registryLabelMap;
299
+ }
300
+ /**
301
+ * Clears cached models
302
+ */
303
+ export async function clearModelCache(mode) {
304
+ const storage = await getStorage();
305
+ if (mode) {
306
+ const modelDir = await getModelPath(mode);
307
+ await storage.rm(modelDir, { recursive: true, force: true });
308
+ }
309
+ else {
310
+ const cacheDir = await getModelCacheDir();
311
+ await storage.rm(cacheDir, { recursive: true, force: true });
312
+ }
313
+ }
314
+ /**
315
+ * Lists downloaded models
316
+ */
317
+ export async function listDownloadedModels() {
318
+ const models = [];
319
+ for (const mode of ["standard", "quantized"]) {
320
+ if (await isModelDownloaded(mode)) {
321
+ const modelPath = await getModelPath(mode);
322
+ const info = MODEL_REGISTRY[mode];
323
+ models.push({ mode, path: modelPath, size: info.size });
324
+ }
325
+ }
326
+ return models;
327
+ }
328
+ /**
329
+ * Gets info about available models
330
+ */
331
+ export function getModelInfo(mode) {
332
+ return MODEL_REGISTRY[mode];
333
+ }
334
+ //# sourceMappingURL=model-manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-manager.js","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAwB,MAAM,UAAU,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAyClD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAgD;IACzE,QAAQ,EAAE;QACR,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,4BAA4B;QAClC,WAAW,EAAE,4DAA4D;QACzE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,wCAAwC;QAChD,WAAW,EAAE,MAAM;QACnB,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE;SACpE;QACD,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR;KACF;IACD,SAAS,EAAE;QACT,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,6BAA6B;QACnC,WAAW,EAAE,2DAA2D;QACxE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,wCAAwC;QAChD,WAAW,EAAE,MAAM;QACnB,KAAK,EAAE;YACL;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,SAAS,EAAE,YAAY;gBACvB,QAAQ,EAAE,IAAI;aACf;SACF;QACD,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR;KACF;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAoB;IACvC,EAAE,QAAQ,EAAE,gBAAgB,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC3E;QACE,QAAQ,EAAE,uBAAuB;QACjC,SAAS,EAAE,uBAAuB;QAClC,QAAQ,EAAE,KAAK;KAChB;IACD;QACE,QAAQ,EAAE,yBAAyB;QACnC,SAAS,EAAE,yBAAyB;QACpC,QAAQ,EAAE,KAAK;KAChB;IACD,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE;CACvE,CAAC;AAEF,0BAA0B;AAC1B,IAAI,eAAe,GAA2B,IAAI,CAAC;AAEnD;;GAEG;AACH,KAAK,UAAU,UAAU;IACvB,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,OAAO,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAA8B;IAE9B,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAC1C,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAA8B;IAE9B,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,CAAC;QACH,6BAA6B;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CACnD,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,MAAM,CACtC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,CACpC,CAAC;YACF,IAAI,CAAC,WAAW;gBAAE,OAAO,KAAK,CAAC;QACjC,CAAC;QAED,4BAA4B;QAC5B,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CACjC,CAAC;QACF,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAYD;;GAEG;AACH,SAAS,iBAAiB,CACxB,IAAY,EACZ,QAAgB,EAChB,SAAkB;IAElB,MAAM,QAAQ,GACZ,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE;QACzC,CAAC,CAAC,GAAG,SAAS,IAAI,QAAQ,EAAE;QAC5B,CAAC,CAAC,QAAQ,CAAC;IACf,OAAO,0BAA0B,IAAI,iBAAiB,QAAQ,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,QAAgB,EAChB,UAAqC;IAErC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE;YACP,YAAY,EAAE,eAAe;SAC9B;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sBAAsB,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC1D,MAAM,KAAK,GACT,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE7E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAC1C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,MAAM,CAAC,IAAI;YAAE,MAAM;QAEvB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAmB,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;QAEhC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC;gBACT,IAAI,EAAE,QAAQ;gBACd,eAAe;gBACf,UAAU,EAAE,KAAK;gBACjB,OAAO,EACL,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC;oBACzB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;oBAC7C,CAAC,CAAC,IAAI;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,GAAW,EACX,QAAgB,EAChB,UAAqC;IAErC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEpC,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/D,MAAM,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAA8B,EAC9B,UAAqC,EACrC,QAAmC;IAEnC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAE1C,mBAAmB;IACnB,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE9B,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,IAAI,2BAA2B,CAAC,CAAC;IAChE,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzC,uBAAuB;IACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhD,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAClE,CAAC;YACJ,CAAC;YACD,mCAAmC;YACnC,QAAQ,EAAE,CAAC,0BAA0B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhD,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAClE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE9E,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IAEjC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAA8B,EAC9B,UAII,EAAE;IAEN,MAAM,EAAE,YAAY,GAAG,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE9D,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,8BAA8B;IAC9B,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEnD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,kBAAkB,QAAQ,OAAO;gBACjD,mCAAmC;gBACnC,sCAAsC,IAAI,gCAAgC;gBAC1E,2BAA2B;gBAC3B,mDAAmD,CACtD,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,QAAQ,EAAE,CAAC,uBAAuB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC;IAEvE,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,IAAI,YAAY,CAAC;QAC/D,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC;QAC3C,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAA8B,EAC9B,YAAoB;IAEpB,MAAM,gBAAgB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;IAEvD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE5C,IACE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,MAAM,GAAG,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,EAClD,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAA+B;IAE/B,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IAEnC,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;QAC1C,MAAM,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IAGxC,MAAM,MAAM,GAIP,EAAE,CAAC;IAER,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,WAAW,CAAU,EAAE,CAAC;QACtD,IAAI,MAAM,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAA8B;IACzD,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
@@ -0,0 +1,172 @@
1
+ /**
2
+ * NER Model Wrapper
3
+ * ONNX Runtime integration for Named Entity Recognition
4
+ * Supports both onnxruntime-node and onnxruntime-web
5
+ */
6
+ import { type OrtSessionOptions } from "#onnx-runtime";
7
+ import { SpanMatch, AnonymizationPolicy } from "../types/index.js";
8
+ /**
9
+ * NER Model configuration
10
+ */
11
+ export interface NERModelConfig {
12
+ /** Path to ONNX model file */
13
+ modelPath: string;
14
+ /** Path to vocabulary file */
15
+ vocabPath: string;
16
+ /** Label mapping (index -> label string) */
17
+ labelMap: string[];
18
+ /** Maximum sequence length */
19
+ maxLength: number;
20
+ /** Number of content tokens shared by adjacent long-input windows. */
21
+ windowOverlapTokens: number;
22
+ /**
23
+ * Maximum number of inference windows per input. Inputs that tokenize to
24
+ * more windows are scanned only up to this bound (onWindowCap fires once per
25
+ * capped inference pass — so twice for one input when caseFallback runs a
26
+ * second pass). Guards callers that feed very large opaque payloads
27
+ * (multi-megabyte base64 attachments) from unbounded inference cost.
28
+ * @default undefined (unbounded — full coverage)
29
+ */
30
+ maxWindowsPerInput?: number;
31
+ /**
32
+ * Called when an input exceeds maxWindowsPerInput and its tail is left
33
+ * unscanned. Callers should log/meter this — it is a coverage reduction,
34
+ * not an error. When absent, a console warning is emitted instead.
35
+ * `inputLength` is the full input character count; the true total window
36
+ * count is not reported because tokenization stops at the cap (computing
37
+ * it would require tokenizing the whole input, the cost the cap avoids).
38
+ */
39
+ onWindowCap?: (info: {
40
+ scannedWindows: number;
41
+ inputLength: number;
42
+ }) => void;
43
+ /** Whether model expects lowercase input */
44
+ doLowerCase: boolean;
45
+ /** Model version for tracking */
46
+ modelVersion: string;
47
+ /** ONNX session options for performance tuning */
48
+ sessionOptions?: OrtSessionOptions;
49
+ /**
50
+ * Enable case-insensitive fallback pass for detecting lowercase names.
51
+ * Runs a second NER pass on title-cased text and merges new detections.
52
+ * @default false
53
+ */
54
+ caseFallback?: boolean;
55
+ /**
56
+ * Confidence penalty multiplier for case-fallback detections (0.0 - 1.0).
57
+ * Applied as: confidence * caseFallbackPenalty
58
+ * @default 0.85
59
+ */
60
+ caseFallbackPenalty?: number;
61
+ }
62
+ /**
63
+ * NER prediction result for a single text
64
+ */
65
+ export interface NERPrediction {
66
+ /** Detected entity spans */
67
+ spans: SpanMatch[];
68
+ /** Processing time in ms */
69
+ processingTimeMs: number;
70
+ /** Model version used */
71
+ modelVersion: string;
72
+ /**
73
+ * True when the input exceeded maxWindowsPerInput and its tail was left
74
+ * unscanned by NER. Callers enforcing a fail-closed policy should treat
75
+ * this as incomplete coverage.
76
+ */
77
+ truncated: boolean;
78
+ /**
79
+ * Number of leading input characters NER actually scanned — the end offset
80
+ * of the last content token. When not truncated this covers the whole input
81
+ * except any trailing whitespace/punctuation past the last token; when
82
+ * truncated, a caller can cut the input at this offset to keep only the
83
+ * fully-scanned portion.
84
+ */
85
+ coverageChars: number;
86
+ }
87
+ /**
88
+ * Default label map for common NER models (CoNLL-style)
89
+ */
90
+ export declare const DEFAULT_LABEL_MAP: string[];
91
+ /**
92
+ * NER Model wrapper for ONNX inference
93
+ */
94
+ export declare class NERModel {
95
+ private ort;
96
+ private session;
97
+ private tokenizer;
98
+ private config;
99
+ private isLoaded;
100
+ constructor(config: NERModelConfig);
101
+ /**
102
+ * Loads the model and tokenizer
103
+ */
104
+ load(): Promise<void>;
105
+ /**
106
+ * Runs a single NER pass: tokenize, infer, decode BIO tags.
107
+ * @param inputText - Text to tokenize and feed to the model
108
+ * @param originalText - Original text used for extracting entity text (may differ in casing)
109
+ */
110
+ private runNERPass;
111
+ /**
112
+ * Predicts entities in text
113
+ */
114
+ predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
115
+ /**
116
+ * Runs ONNX inference
117
+ */
118
+ private runInference;
119
+ /**
120
+ * Processes model logits to extract labels and confidences
121
+ */
122
+ private processLogits;
123
+ /**
124
+ * Gets minimum confidence threshold from policy
125
+ */
126
+ private getMinConfidence;
127
+ /**
128
+ * Gets model version
129
+ */
130
+ get version(): string;
131
+ /**
132
+ * Checks if model is loaded
133
+ */
134
+ get loaded(): boolean;
135
+ /**
136
+ * Disposes of model resources
137
+ */
138
+ dispose(): Promise<void>;
139
+ }
140
+ /**
141
+ * Creates a NER model instance with configuration
142
+ */
143
+ export declare function createNERModel(config: Partial<NERModelConfig> & {
144
+ modelPath: string;
145
+ vocabPath: string;
146
+ }): NERModel;
147
+ /**
148
+ * NER Model stub for when no model is available
149
+ * Returns empty results - useful for regex-only mode
150
+ */
151
+ export declare class NERModelStub {
152
+ readonly version = "stub-1.0.0";
153
+ readonly loaded = true;
154
+ load(): Promise<void>;
155
+ predict(_text: string, _policy?: AnonymizationPolicy): Promise<NERPrediction>;
156
+ dispose(): Promise<void>;
157
+ }
158
+ /**
159
+ * Creates a stub NER model (for testing or regex-only mode)
160
+ */
161
+ export declare function createNERModelStub(): NERModelStub;
162
+ /**
163
+ * NER model interface for dependency injection
164
+ */
165
+ export interface INERModel {
166
+ readonly version: string;
167
+ readonly loaded: boolean;
168
+ load(): Promise<void>;
169
+ predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
170
+ dispose(): Promise<void>;
171
+ }
172
+ //# sourceMappingURL=ner-model.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ner-model.d.ts","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAoBnE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAC9E,4CAA4C;IAC5C,WAAW,EAAE,OAAO,CAAC;IACrB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,cAAc,CAAC,EAAE,iBAAiB,CAAC;IACnC;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,4BAA4B;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;IACnB;;;;;;OAMG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,UAU7B,CAAC;AA8BF;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,GAAG,CAA2B;IACtC,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C3B;;;;OAIG;YACW,UAAU;IAkExB;;OAEG;IACG,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC;IAmEzB;;OAEG;YACW,YAAY;IAiE1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqCrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAexB;;OAEG;IACH,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAOzB;AAsKD;;GAEG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzE,QAAQ,CA0BV;AAED;;;GAGG;AACH,qBAAa,YAAY;IACvB,QAAQ,CAAC,OAAO,gBAAgB;IAChC,QAAQ,CAAC,MAAM,QAAQ;IAEjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,OAAO,CACL,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,aAAa,CAAC;IAUzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAIzB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,YAAY,CAEjD;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5E,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}