opencode-rehydra-core 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/LICENSE +22 -0
  2. package/README.en.md +191 -0
  3. package/README.md +191 -0
  4. package/dist/browser.d.ts +54 -0
  5. package/dist/browser.d.ts.map +1 -0
  6. package/dist/browser.js +74 -0
  7. package/dist/browser.js.map +1 -0
  8. package/dist/core/anonymizer.d.ts +309 -0
  9. package/dist/core/anonymizer.d.ts.map +1 -0
  10. package/dist/core/anonymizer.js +555 -0
  11. package/dist/core/anonymizer.js.map +1 -0
  12. package/dist/core/index.d.ts +6 -0
  13. package/dist/core/index.d.ts.map +1 -0
  14. package/dist/core/index.js +6 -0
  15. package/dist/core/index.js.map +1 -0
  16. package/dist/crypto/index.d.ts +6 -0
  17. package/dist/crypto/index.d.ts.map +1 -0
  18. package/dist/crypto/index.js +6 -0
  19. package/dist/crypto/index.js.map +1 -0
  20. package/dist/crypto/pii-map-crypto.d.ts +114 -0
  21. package/dist/crypto/pii-map-crypto.d.ts.map +1 -0
  22. package/dist/crypto/pii-map-crypto.js +228 -0
  23. package/dist/crypto/pii-map-crypto.js.map +1 -0
  24. package/dist/index.d.ts +53 -0
  25. package/dist/index.d.ts.map +1 -0
  26. package/dist/index.js +72 -0
  27. package/dist/index.js.map +1 -0
  28. package/dist/ner/bio-decoder.d.ts +64 -0
  29. package/dist/ner/bio-decoder.d.ts.map +1 -0
  30. package/dist/ner/bio-decoder.js +218 -0
  31. package/dist/ner/bio-decoder.js.map +1 -0
  32. package/dist/ner/index.d.ts +12 -0
  33. package/dist/ner/index.d.ts.map +1 -0
  34. package/dist/ner/index.js +12 -0
  35. package/dist/ner/index.js.map +1 -0
  36. package/dist/ner/inference-server-client.d.ts +81 -0
  37. package/dist/ner/inference-server-client.d.ts.map +1 -0
  38. package/dist/ner/inference-server-client.js +120 -0
  39. package/dist/ner/inference-server-client.js.map +1 -0
  40. package/dist/ner/inference-server-ner-model.d.ts +43 -0
  41. package/dist/ner/inference-server-ner-model.d.ts.map +1 -0
  42. package/dist/ner/inference-server-ner-model.js +119 -0
  43. package/dist/ner/inference-server-ner-model.js.map +1 -0
  44. package/dist/ner/model-manager.d.ts +111 -0
  45. package/dist/ner/model-manager.d.ts.map +1 -0
  46. package/dist/ner/model-manager.js +334 -0
  47. package/dist/ner/model-manager.js.map +1 -0
  48. package/dist/ner/ner-model.d.ts +172 -0
  49. package/dist/ner/ner-model.d.ts.map +1 -0
  50. package/dist/ner/ner-model.js +514 -0
  51. package/dist/ner/ner-model.js.map +1 -0
  52. package/dist/ner/onnx-runtime.browser.d.ts +46 -0
  53. package/dist/ner/onnx-runtime.browser.d.ts.map +1 -0
  54. package/dist/ner/onnx-runtime.browser.js +90 -0
  55. package/dist/ner/onnx-runtime.browser.js.map +1 -0
  56. package/dist/ner/onnx-runtime.d.ts +72 -0
  57. package/dist/ner/onnx-runtime.d.ts.map +1 -0
  58. package/dist/ner/onnx-runtime.js +132 -0
  59. package/dist/ner/onnx-runtime.js.map +1 -0
  60. package/dist/ner/token-window.d.ts +19 -0
  61. package/dist/ner/token-window.d.ts.map +1 -0
  62. package/dist/ner/token-window.js +101 -0
  63. package/dist/ner/token-window.js.map +1 -0
  64. package/dist/ner/tokenizer.d.ts +142 -0
  65. package/dist/ner/tokenizer.d.ts.map +1 -0
  66. package/dist/ner/tokenizer.js +362 -0
  67. package/dist/ner/tokenizer.js.map +1 -0
  68. package/dist/opencode-plugin/git-identity.d.ts +3 -0
  69. package/dist/opencode-plugin/git-identity.d.ts.map +1 -0
  70. package/dist/opencode-plugin/git-identity.js +37 -0
  71. package/dist/opencode-plugin/git-identity.js.map +1 -0
  72. package/dist/opencode-plugin/github-username.d.ts +3 -0
  73. package/dist/opencode-plugin/github-username.d.ts.map +1 -0
  74. package/dist/opencode-plugin/github-username.js +41 -0
  75. package/dist/opencode-plugin/github-username.js.map +1 -0
  76. package/dist/opencode-plugin/identity-recognizer.d.ts +5 -0
  77. package/dist/opencode-plugin/identity-recognizer.d.ts.map +1 -0
  78. package/dist/opencode-plugin/identity-recognizer.js +21 -0
  79. package/dist/opencode-plugin/identity-recognizer.js.map +1 -0
  80. package/dist/opencode-plugin/index.d.ts +7 -0
  81. package/dist/opencode-plugin/index.d.ts.map +1 -0
  82. package/dist/opencode-plugin/index.js +6 -0
  83. package/dist/opencode-plugin/index.js.map +1 -0
  84. package/dist/opencode-plugin/plugin.d.ts +39 -0
  85. package/dist/opencode-plugin/plugin.d.ts.map +1 -0
  86. package/dist/opencode-plugin/plugin.js +294 -0
  87. package/dist/opencode-plugin/plugin.js.map +1 -0
  88. package/dist/opencode-plugin/types.d.ts +33 -0
  89. package/dist/opencode-plugin/types.d.ts.map +1 -0
  90. package/dist/opencode-plugin/types.js +5 -0
  91. package/dist/opencode-plugin/types.js.map +1 -0
  92. package/dist/opencode-plugin/v2/anon-state.d.ts +12 -0
  93. package/dist/opencode-plugin/v2/anon-state.d.ts.map +1 -0
  94. package/dist/opencode-plugin/v2/anon-state.js +14 -0
  95. package/dist/opencode-plugin/v2/anon-state.js.map +1 -0
  96. package/dist/opencode-plugin/v2/deep-rehydrate.d.ts +13 -0
  97. package/dist/opencode-plugin/v2/deep-rehydrate.d.ts.map +1 -0
  98. package/dist/opencode-plugin/v2/deep-rehydrate.js +33 -0
  99. package/dist/opencode-plugin/v2/deep-rehydrate.js.map +1 -0
  100. package/dist/opencode-plugin/v2/index.d.ts +17 -0
  101. package/dist/opencode-plugin/v2/index.d.ts.map +1 -0
  102. package/dist/opencode-plugin/v2/index.js +17 -0
  103. package/dist/opencode-plugin/v2/index.js.map +1 -0
  104. package/dist/opencode-plugin/v2/logger.d.ts +24 -0
  105. package/dist/opencode-plugin/v2/logger.d.ts.map +1 -0
  106. package/dist/opencode-plugin/v2/logger.js +73 -0
  107. package/dist/opencode-plugin/v2/logger.js.map +1 -0
  108. package/dist/opencode-plugin/v2/options.d.ts +40 -0
  109. package/dist/opencode-plugin/v2/options.d.ts.map +1 -0
  110. package/dist/opencode-plugin/v2/options.js +132 -0
  111. package/dist/opencode-plugin/v2/options.js.map +1 -0
  112. package/dist/opencode-plugin/v2/register-hooks.d.ts +43 -0
  113. package/dist/opencode-plugin/v2/register-hooks.d.ts.map +1 -0
  114. package/dist/opencode-plugin/v2/register-hooks.js +160 -0
  115. package/dist/opencode-plugin/v2/register-hooks.js.map +1 -0
  116. package/dist/opencode-plugin/v2/rehydrate-tail.d.ts +101 -0
  117. package/dist/opencode-plugin/v2/rehydrate-tail.d.ts.map +1 -0
  118. package/dist/opencode-plugin/v2/rehydrate-tail.js +147 -0
  119. package/dist/opencode-plugin/v2/rehydrate-tail.js.map +1 -0
  120. package/dist/opencode-plugin/v2/response-rehydrate.d.ts +78 -0
  121. package/dist/opencode-plugin/v2/response-rehydrate.d.ts.map +1 -0
  122. package/dist/opencode-plugin/v2/response-rehydrate.js +210 -0
  123. package/dist/opencode-plugin/v2/response-rehydrate.js.map +1 -0
  124. package/dist/opencode-plugin/v2/scrub-messages.d.ts +39 -0
  125. package/dist/opencode-plugin/v2/scrub-messages.d.ts.map +1 -0
  126. package/dist/opencode-plugin/v2/scrub-messages.js +260 -0
  127. package/dist/opencode-plugin/v2/scrub-messages.js.map +1 -0
  128. package/dist/opencode-plugin/v2/sessions.d.ts +17 -0
  129. package/dist/opencode-plugin/v2/sessions.d.ts.map +1 -0
  130. package/dist/opencode-plugin/v2/sessions.js +23 -0
  131. package/dist/opencode-plugin/v2/sessions.js.map +1 -0
  132. package/dist/opencode-plugin/v2/sse.d.ts +101 -0
  133. package/dist/opencode-plugin/v2/sse.d.ts.map +1 -0
  134. package/dist/opencode-plugin/v2/sse.js +235 -0
  135. package/dist/opencode-plugin/v2/sse.js.map +1 -0
  136. package/dist/opencode-plugin/v2/types.d.ts +188 -0
  137. package/dist/opencode-plugin/v2/types.d.ts.map +1 -0
  138. package/dist/opencode-plugin/v2/types.js +10 -0
  139. package/dist/opencode-plugin/v2/types.js.map +1 -0
  140. package/dist/opencode-plugin/vcs-command.d.ts +3 -0
  141. package/dist/opencode-plugin/vcs-command.d.ts.map +1 -0
  142. package/dist/opencode-plugin/vcs-command.js +34 -0
  143. package/dist/opencode-plugin/vcs-command.js.map +1 -0
  144. package/dist/pipeline/index.d.ts +12 -0
  145. package/dist/pipeline/index.d.ts.map +1 -0
  146. package/dist/pipeline/index.js +12 -0
  147. package/dist/pipeline/index.js.map +1 -0
  148. package/dist/pipeline/prenormalize.d.ts +48 -0
  149. package/dist/pipeline/prenormalize.d.ts.map +1 -0
  150. package/dist/pipeline/prenormalize.js +94 -0
  151. package/dist/pipeline/prenormalize.js.map +1 -0
  152. package/dist/pipeline/resolver.d.ts +56 -0
  153. package/dist/pipeline/resolver.d.ts.map +1 -0
  154. package/dist/pipeline/resolver.js +239 -0
  155. package/dist/pipeline/resolver.js.map +1 -0
  156. package/dist/pipeline/semantic-data-loader.d.ts +171 -0
  157. package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
  158. package/dist/pipeline/semantic-data-loader.js +677 -0
  159. package/dist/pipeline/semantic-data-loader.js.map +1 -0
  160. package/dist/pipeline/semantic-enricher.d.ts +112 -0
  161. package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
  162. package/dist/pipeline/semantic-enricher.js +322 -0
  163. package/dist/pipeline/semantic-enricher.js.map +1 -0
  164. package/dist/pipeline/tagger.d.ts +121 -0
  165. package/dist/pipeline/tagger.d.ts.map +1 -0
  166. package/dist/pipeline/tagger.js +545 -0
  167. package/dist/pipeline/tagger.js.map +1 -0
  168. package/dist/pipeline/title-extractor.d.ts +79 -0
  169. package/dist/pipeline/title-extractor.d.ts.map +1 -0
  170. package/dist/pipeline/title-extractor.js +801 -0
  171. package/dist/pipeline/title-extractor.js.map +1 -0
  172. package/dist/pipeline/validator.d.ts +65 -0
  173. package/dist/pipeline/validator.d.ts.map +1 -0
  174. package/dist/pipeline/validator.js +292 -0
  175. package/dist/pipeline/validator.js.map +1 -0
  176. package/dist/proxy/index.d.ts +13 -0
  177. package/dist/proxy/index.d.ts.map +1 -0
  178. package/dist/proxy/index.js +12 -0
  179. package/dist/proxy/index.js.map +1 -0
  180. package/dist/proxy/providers/anthropic.d.ts +28 -0
  181. package/dist/proxy/providers/anthropic.d.ts.map +1 -0
  182. package/dist/proxy/providers/anthropic.js +265 -0
  183. package/dist/proxy/providers/anthropic.js.map +1 -0
  184. package/dist/proxy/providers/index.d.ts +19 -0
  185. package/dist/proxy/providers/index.d.ts.map +1 -0
  186. package/dist/proxy/providers/index.js +41 -0
  187. package/dist/proxy/providers/index.js.map +1 -0
  188. package/dist/proxy/providers/openai.d.ts +27 -0
  189. package/dist/proxy/providers/openai.d.ts.map +1 -0
  190. package/dist/proxy/providers/openai.js +203 -0
  191. package/dist/proxy/providers/openai.js.map +1 -0
  192. package/dist/proxy/providers/streaming-tool-loop.d.ts +12 -0
  193. package/dist/proxy/providers/streaming-tool-loop.d.ts.map +1 -0
  194. package/dist/proxy/providers/streaming-tool-loop.js +223 -0
  195. package/dist/proxy/providers/streaming-tool-loop.js.map +1 -0
  196. package/dist/proxy/providers/types.d.ts +109 -0
  197. package/dist/proxy/providers/types.d.ts.map +1 -0
  198. package/dist/proxy/providers/types.js +2 -0
  199. package/dist/proxy/providers/types.js.map +1 -0
  200. package/dist/proxy/proxy-server.d.ts +61 -0
  201. package/dist/proxy/proxy-server.d.ts.map +1 -0
  202. package/dist/proxy/proxy-server.js +146 -0
  203. package/dist/proxy/proxy-server.js.map +1 -0
  204. package/dist/proxy/rehydra-fetch.d.ts +35 -0
  205. package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
  206. package/dist/proxy/rehydra-fetch.js +631 -0
  207. package/dist/proxy/rehydra-fetch.js.map +1 -0
  208. package/dist/proxy/rehydra-proxy.d.ts +40 -0
  209. package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
  210. package/dist/proxy/rehydra-proxy.js +150 -0
  211. package/dist/proxy/rehydra-proxy.js.map +1 -0
  212. package/dist/proxy/sse-parser.d.ts +59 -0
  213. package/dist/proxy/sse-parser.d.ts.map +1 -0
  214. package/dist/proxy/sse-parser.js +112 -0
  215. package/dist/proxy/sse-parser.js.map +1 -0
  216. package/dist/proxy/system-instruction.d.ts +19 -0
  217. package/dist/proxy/system-instruction.d.ts.map +1 -0
  218. package/dist/proxy/system-instruction.js +33 -0
  219. package/dist/proxy/system-instruction.js.map +1 -0
  220. package/dist/proxy/tool-loop-stream.d.ts +7 -0
  221. package/dist/proxy/tool-loop-stream.d.ts.map +1 -0
  222. package/dist/proxy/tool-loop-stream.js +65 -0
  223. package/dist/proxy/tool-loop-stream.js.map +1 -0
  224. package/dist/proxy/types.d.ts +138 -0
  225. package/dist/proxy/types.d.ts.map +1 -0
  226. package/dist/proxy/types.js +5 -0
  227. package/dist/proxy/types.js.map +1 -0
  228. package/dist/proxy/wrap-client.d.ts +47 -0
  229. package/dist/proxy/wrap-client.d.ts.map +1 -0
  230. package/dist/proxy/wrap-client.js +70 -0
  231. package/dist/proxy/wrap-client.js.map +1 -0
  232. package/dist/recognizers/address.d.ts +2 -0
  233. package/dist/recognizers/address.d.ts.map +1 -0
  234. package/dist/recognizers/address.js +23 -0
  235. package/dist/recognizers/address.js.map +1 -0
  236. package/dist/recognizers/base.d.ts +78 -0
  237. package/dist/recognizers/base.d.ts.map +1 -0
  238. package/dist/recognizers/base.js +100 -0
  239. package/dist/recognizers/base.js.map +1 -0
  240. package/dist/recognizers/bic-swift.d.ts +10 -0
  241. package/dist/recognizers/bic-swift.d.ts.map +1 -0
  242. package/dist/recognizers/bic-swift.js +107 -0
  243. package/dist/recognizers/bic-swift.js.map +1 -0
  244. package/dist/recognizers/credit-card.d.ts +32 -0
  245. package/dist/recognizers/credit-card.d.ts.map +1 -0
  246. package/dist/recognizers/credit-card.js +160 -0
  247. package/dist/recognizers/credit-card.js.map +1 -0
  248. package/dist/recognizers/custom-id.d.ts +28 -0
  249. package/dist/recognizers/custom-id.d.ts.map +1 -0
  250. package/dist/recognizers/custom-id.js +116 -0
  251. package/dist/recognizers/custom-id.js.map +1 -0
  252. package/dist/recognizers/date.d.ts +10 -0
  253. package/dist/recognizers/date.d.ts.map +1 -0
  254. package/dist/recognizers/date.js +228 -0
  255. package/dist/recognizers/date.js.map +1 -0
  256. package/dist/recognizers/email.d.ts +10 -0
  257. package/dist/recognizers/email.d.ts.map +1 -0
  258. package/dist/recognizers/email.js +75 -0
  259. package/dist/recognizers/email.js.map +1 -0
  260. package/dist/recognizers/iban.d.ts +14 -0
  261. package/dist/recognizers/iban.d.ts.map +1 -0
  262. package/dist/recognizers/iban.js +67 -0
  263. package/dist/recognizers/iban.js.map +1 -0
  264. package/dist/recognizers/index.d.ts +25 -0
  265. package/dist/recognizers/index.d.ts.map +1 -0
  266. package/dist/recognizers/index.js +53 -0
  267. package/dist/recognizers/index.js.map +1 -0
  268. package/dist/recognizers/ip-address.d.ts +14 -0
  269. package/dist/recognizers/ip-address.d.ts.map +1 -0
  270. package/dist/recognizers/ip-address.js +183 -0
  271. package/dist/recognizers/ip-address.js.map +1 -0
  272. package/dist/recognizers/phone.d.ts +10 -0
  273. package/dist/recognizers/phone.d.ts.map +1 -0
  274. package/dist/recognizers/phone.js +145 -0
  275. package/dist/recognizers/phone.js.map +1 -0
  276. package/dist/recognizers/postal-code.d.ts +16 -0
  277. package/dist/recognizers/postal-code.d.ts.map +1 -0
  278. package/dist/recognizers/postal-code.js +42 -0
  279. package/dist/recognizers/postal-code.js.map +1 -0
  280. package/dist/recognizers/registry.d.ts +59 -0
  281. package/dist/recognizers/registry.d.ts.map +1 -0
  282. package/dist/recognizers/registry.js +113 -0
  283. package/dist/recognizers/registry.js.map +1 -0
  284. package/dist/recognizers/secrets/api-key.d.ts +6 -0
  285. package/dist/recognizers/secrets/api-key.d.ts.map +1 -0
  286. package/dist/recognizers/secrets/api-key.js +40 -0
  287. package/dist/recognizers/secrets/api-key.js.map +1 -0
  288. package/dist/recognizers/secrets/aws-credentials.d.ts +7 -0
  289. package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -0
  290. package/dist/recognizers/secrets/aws-credentials.js +68 -0
  291. package/dist/recognizers/secrets/aws-credentials.js.map +1 -0
  292. package/dist/recognizers/secrets/config-secret.d.ts +8 -0
  293. package/dist/recognizers/secrets/config-secret.d.ts.map +1 -0
  294. package/dist/recognizers/secrets/config-secret.js +59 -0
  295. package/dist/recognizers/secrets/config-secret.js.map +1 -0
  296. package/dist/recognizers/secrets/connection-string.d.ts +6 -0
  297. package/dist/recognizers/secrets/connection-string.d.ts.map +1 -0
  298. package/dist/recognizers/secrets/connection-string.js +40 -0
  299. package/dist/recognizers/secrets/connection-string.js.map +1 -0
  300. package/dist/recognizers/secrets/env-var.d.ts +8 -0
  301. package/dist/recognizers/secrets/env-var.d.ts.map +1 -0
  302. package/dist/recognizers/secrets/env-var.js +54 -0
  303. package/dist/recognizers/secrets/env-var.js.map +1 -0
  304. package/dist/recognizers/secrets/index.d.ts +22 -0
  305. package/dist/recognizers/secrets/index.d.ts.map +1 -0
  306. package/dist/recognizers/secrets/index.js +41 -0
  307. package/dist/recognizers/secrets/index.js.map +1 -0
  308. package/dist/recognizers/secrets/jwt.d.ts +6 -0
  309. package/dist/recognizers/secrets/jwt.d.ts.map +1 -0
  310. package/dist/recognizers/secrets/jwt.js +47 -0
  311. package/dist/recognizers/secrets/jwt.js.map +1 -0
  312. package/dist/recognizers/secrets/key-patterns.d.ts +10 -0
  313. package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -0
  314. package/dist/recognizers/secrets/key-patterns.js +15 -0
  315. package/dist/recognizers/secrets/key-patterns.js.map +1 -0
  316. package/dist/recognizers/secrets/literal-value.d.ts +10 -0
  317. package/dist/recognizers/secrets/literal-value.d.ts.map +1 -0
  318. package/dist/recognizers/secrets/literal-value.js +48 -0
  319. package/dist/recognizers/secrets/literal-value.js.map +1 -0
  320. package/dist/recognizers/secrets/private-key.d.ts +6 -0
  321. package/dist/recognizers/secrets/private-key.d.ts.map +1 -0
  322. package/dist/recognizers/secrets/private-key.js +16 -0
  323. package/dist/recognizers/secrets/private-key.js.map +1 -0
  324. package/dist/recognizers/url.d.ts +14 -0
  325. package/dist/recognizers/url.d.ts.map +1 -0
  326. package/dist/recognizers/url.js +121 -0
  327. package/dist/recognizers/url.js.map +1 -0
  328. package/dist/storage/browser.d.ts +13 -0
  329. package/dist/storage/browser.d.ts.map +1 -0
  330. package/dist/storage/browser.js +13 -0
  331. package/dist/storage/browser.js.map +1 -0
  332. package/dist/storage/in-memory.d.ts +60 -0
  333. package/dist/storage/in-memory.d.ts.map +1 -0
  334. package/dist/storage/in-memory.js +108 -0
  335. package/dist/storage/in-memory.js.map +1 -0
  336. package/dist/storage/index.d.ts +11 -0
  337. package/dist/storage/index.d.ts.map +1 -0
  338. package/dist/storage/index.js +11 -0
  339. package/dist/storage/index.js.map +1 -0
  340. package/dist/storage/indexeddb.d.ts +69 -0
  341. package/dist/storage/indexeddb.d.ts.map +1 -0
  342. package/dist/storage/indexeddb.js +274 -0
  343. package/dist/storage/indexeddb.js.map +1 -0
  344. package/dist/storage/session-base.d.ts +34 -0
  345. package/dist/storage/session-base.d.ts.map +1 -0
  346. package/dist/storage/session-base.js +122 -0
  347. package/dist/storage/session-base.js.map +1 -0
  348. package/dist/storage/session.d.ts +19 -0
  349. package/dist/storage/session.d.ts.map +1 -0
  350. package/dist/storage/session.js +31 -0
  351. package/dist/storage/session.js.map +1 -0
  352. package/dist/storage/sqlite.d.ts +80 -0
  353. package/dist/storage/sqlite.d.ts.map +1 -0
  354. package/dist/storage/sqlite.js +255 -0
  355. package/dist/storage/sqlite.js.map +1 -0
  356. package/dist/storage/types.d.ts +208 -0
  357. package/dist/storage/types.d.ts.map +1 -0
  358. package/dist/storage/types.js +6 -0
  359. package/dist/storage/types.js.map +1 -0
  360. package/dist/streaming/anonymizer-stream.d.ts +63 -0
  361. package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
  362. package/dist/streaming/anonymizer-stream.js +184 -0
  363. package/dist/streaming/anonymizer-stream.js.map +1 -0
  364. package/dist/streaming/index.d.ts +9 -0
  365. package/dist/streaming/index.d.ts.map +1 -0
  366. package/dist/streaming/index.js +8 -0
  367. package/dist/streaming/index.js.map +1 -0
  368. package/dist/streaming/sentence-buffer.d.ts +78 -0
  369. package/dist/streaming/sentence-buffer.d.ts.map +1 -0
  370. package/dist/streaming/sentence-buffer.js +239 -0
  371. package/dist/streaming/sentence-buffer.js.map +1 -0
  372. package/dist/streaming/stream-factory.d.ts +38 -0
  373. package/dist/streaming/stream-factory.d.ts.map +1 -0
  374. package/dist/streaming/stream-factory.js +69 -0
  375. package/dist/streaming/stream-factory.js.map +1 -0
  376. package/dist/streaming/types.d.ts +121 -0
  377. package/dist/streaming/types.d.ts.map +1 -0
  378. package/dist/streaming/types.js +5 -0
  379. package/dist/streaming/types.js.map +1 -0
  380. package/dist/types/index.d.ts +269 -0
  381. package/dist/types/index.d.ts.map +1 -0
  382. package/dist/types/index.js +95 -0
  383. package/dist/types/index.js.map +1 -0
  384. package/dist/types/pii-types.d.ts +64 -0
  385. package/dist/types/pii-types.d.ts.map +1 -0
  386. package/dist/types/pii-types.js +153 -0
  387. package/dist/types/pii-types.js.map +1 -0
  388. package/dist/utils/iban-checksum.d.ts +23 -0
  389. package/dist/utils/iban-checksum.d.ts.map +1 -0
  390. package/dist/utils/iban-checksum.js +106 -0
  391. package/dist/utils/iban-checksum.js.map +1 -0
  392. package/dist/utils/index.d.ts +11 -0
  393. package/dist/utils/index.d.ts.map +1 -0
  394. package/dist/utils/index.js +11 -0
  395. package/dist/utils/index.js.map +1 -0
  396. package/dist/utils/json-walk.d.ts +21 -0
  397. package/dist/utils/json-walk.d.ts.map +1 -0
  398. package/dist/utils/json-walk.js +44 -0
  399. package/dist/utils/json-walk.js.map +1 -0
  400. package/dist/utils/luhn.d.ts +17 -0
  401. package/dist/utils/luhn.d.ts.map +1 -0
  402. package/dist/utils/luhn.js +55 -0
  403. package/dist/utils/luhn.js.map +1 -0
  404. package/dist/utils/offsets.d.ts +86 -0
  405. package/dist/utils/offsets.d.ts.map +1 -0
  406. package/dist/utils/offsets.js +124 -0
  407. package/dist/utils/offsets.js.map +1 -0
  408. package/dist/utils/path.d.ts +34 -0
  409. package/dist/utils/path.d.ts.map +1 -0
  410. package/dist/utils/path.js +96 -0
  411. package/dist/utils/path.js.map +1 -0
  412. package/dist/utils/regex.d.ts +11 -0
  413. package/dist/utils/regex.d.ts.map +1 -0
  414. package/dist/utils/regex.js +14 -0
  415. package/dist/utils/regex.js.map +1 -0
  416. package/dist/utils/storage-browser.d.ts +51 -0
  417. package/dist/utils/storage-browser.d.ts.map +1 -0
  418. package/dist/utils/storage-browser.js +381 -0
  419. package/dist/utils/storage-browser.js.map +1 -0
  420. package/dist/utils/storage-node.d.ts +51 -0
  421. package/dist/utils/storage-node.d.ts.map +1 -0
  422. package/dist/utils/storage-node.js +153 -0
  423. package/dist/utils/storage-node.js.map +1 -0
  424. package/dist/utils/storage.browser.d.ts +75 -0
  425. package/dist/utils/storage.browser.d.ts.map +1 -0
  426. package/dist/utils/storage.browser.js +69 -0
  427. package/dist/utils/storage.browser.js.map +1 -0
  428. package/dist/utils/storage.d.ts +73 -0
  429. package/dist/utils/storage.d.ts.map +1 -0
  430. package/dist/utils/storage.js +69 -0
  431. package/dist/utils/storage.js.map +1 -0
  432. package/package.json +148 -0
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Pipeline Module
3
+ * Exports all pipeline components
4
+ */
5
+ export * from "./prenormalize.js";
6
+ export * from "./resolver.js";
7
+ export * from "./tagger.js";
8
+ export * from "./validator.js";
9
+ export * from "./semantic-enricher.js";
10
+ export * from "./semantic-data-loader.js";
11
+ export * from "./title-extractor.js";
12
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sBAAsB,CAAC"}
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Pre-normalization
3
+ * Normalizes text before PII detection
4
+ */
5
+ /**
6
+ * Pre-normalization options
7
+ */
8
+ export interface PrenormalizeOptions {
9
+ /** Normalize line endings to \n */
10
+ normalizeLineEndings: boolean;
11
+ /** Apply Unicode NFKC normalization */
12
+ unicodeNormalize: boolean;
13
+ /** Trim leading/trailing whitespace */
14
+ trim: boolean;
15
+ }
16
+ /**
17
+ * Default pre-normalization options
18
+ */
19
+ export declare const DEFAULT_PRENORMALIZE_OPTIONS: PrenormalizeOptions;
20
+ /**
21
+ * Pre-normalizes text for PII detection
22
+ * Note: This currently only normalizes line endings to preserve character offsets
23
+ *
24
+ * @param text - Original input text
25
+ * @param options - Normalization options
26
+ * @returns Normalized text
27
+ */
28
+ export declare function prenormalize(text: string, options?: Partial<PrenormalizeOptions>): string;
29
+ /**
30
+ * Calculates offset adjustments when text is modified
31
+ * Used when prenormalization changes text length
32
+ */
33
+ export interface OffsetMapping {
34
+ /** Map from original offset to normalized offset */
35
+ toNormalized: (originalOffset: number) => number;
36
+ /** Map from normalized offset to original offset */
37
+ toOriginal: (normalizedOffset: number) => number;
38
+ }
39
+ /**
40
+ * Creates an identity offset mapping (no changes)
41
+ */
42
+ export declare function createIdentityMapping(): OffsetMapping;
43
+ /**
44
+ * Creates offset mapping for line ending normalization
45
+ * This handles \r\n -> \n replacement
46
+ */
47
+ export declare function createLineEndingMapping(originalText: string): OffsetMapping;
48
+ //# sourceMappingURL=prenormalize.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"prenormalize.d.ts","sourceRoot":"","sources":["../../src/pipeline/prenormalize.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,uCAAuC;IACvC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,uCAAuC;IACvC,IAAI,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,mBAI1C,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM,GACzC,MAAM,CAqBR;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,YAAY,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,MAAM,CAAC;IACjD,oDAAoD;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,aAAa,CAKrD;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,aAAa,CA4C3E"}
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Pre-normalization
3
+ * Normalizes text before PII detection
4
+ */
5
+ /**
6
+ * Default pre-normalization options
7
+ */
8
+ export const DEFAULT_PRENORMALIZE_OPTIONS = {
9
+ normalizeLineEndings: true,
10
+ unicodeNormalize: false, // Disabled by default to preserve offsets
11
+ trim: false, // Disabled by default to preserve offsets
12
+ };
13
+ /**
14
+ * Pre-normalizes text for PII detection
15
+ * Note: This currently only normalizes line endings to preserve character offsets
16
+ *
17
+ * @param text - Original input text
18
+ * @param options - Normalization options
19
+ * @returns Normalized text
20
+ */
21
+ export function prenormalize(text, options = {}) {
22
+ const opts = { ...DEFAULT_PRENORMALIZE_OPTIONS, ...options };
23
+ let result = text;
24
+ // Normalize line endings (\r\n -> \n, \r -> \n)
25
+ if (opts.normalizeLineEndings) {
26
+ result = result.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
27
+ }
28
+ // Note: Unicode normalization (NFKC) can change string length
29
+ // We skip it by default to preserve character offsets
30
+ // If needed, implement offset mapping
31
+ if (opts.unicodeNormalize) {
32
+ result = result.normalize('NFKC');
33
+ }
34
+ if (opts.trim) {
35
+ result = result.trim();
36
+ }
37
+ return result;
38
+ }
39
+ /**
40
+ * Creates an identity offset mapping (no changes)
41
+ */
42
+ export function createIdentityMapping() {
43
+ return {
44
+ toNormalized: (offset) => offset,
45
+ toOriginal: (offset) => offset,
46
+ };
47
+ }
48
+ /**
49
+ * Creates offset mapping for line ending normalization
50
+ * This handles \r\n -> \n replacement
51
+ */
52
+ export function createLineEndingMapping(originalText) {
53
+ // Find all \r\n positions
54
+ const crlfPositions = [];
55
+ for (let i = 0; i < originalText.length - 1; i++) {
56
+ if (originalText[i] === '\r' && originalText[i + 1] === '\n') {
57
+ crlfPositions.push(i);
58
+ }
59
+ }
60
+ if (crlfPositions.length === 0) {
61
+ return createIdentityMapping();
62
+ }
63
+ return {
64
+ toNormalized(originalOffset) {
65
+ // Count how many \r\n pairs are before this offset
66
+ let adjustment = 0;
67
+ for (const pos of crlfPositions) {
68
+ if (pos < originalOffset) {
69
+ adjustment++;
70
+ }
71
+ else {
72
+ break;
73
+ }
74
+ }
75
+ return originalOffset - adjustment;
76
+ },
77
+ toOriginal(normalizedOffset) {
78
+ // Add back the removed \r characters
79
+ let adjustment = 0;
80
+ let currentNormalized = 0;
81
+ for (const pos of crlfPositions) {
82
+ if (currentNormalized + (pos - adjustment) <= normalizedOffset) {
83
+ adjustment++;
84
+ currentNormalized = pos - adjustment + 1;
85
+ }
86
+ else {
87
+ break;
88
+ }
89
+ }
90
+ return normalizedOffset + adjustment;
91
+ },
92
+ };
93
+ }
94
+ //# sourceMappingURL=prenormalize.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"prenormalize.js","sourceRoot":"","sources":["../../src/pipeline/prenormalize.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH;;GAEG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAwB;IAC/D,oBAAoB,EAAE,IAAI;IAC1B,gBAAgB,EAAE,KAAK,EAAE,0CAA0C;IACnE,IAAI,EAAE,KAAK,EAAE,0CAA0C;CACxD,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,UAAwC,EAAE;IAE1C,MAAM,IAAI,GAAG,EAAE,GAAG,4BAA4B,EAAE,GAAG,OAAO,EAAE,CAAC;IAC7D,IAAI,MAAM,GAAG,IAAI,CAAC;IAElB,gDAAgD;IAChD,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC9B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,8DAA8D;IAC9D,sDAAsD;IACtD,sCAAsC;IACtC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,OAAO;QACL,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;QAChC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAAoB;IAC1D,0BAA0B;IAC1B,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7D,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,qBAAqB,EAAE,CAAC;IACjC,CAAC;IAED,OAAO;QACL,YAAY,CAAC,cAAsB;YACjC,mDAAmD;YACnD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,GAAG,GAAG,cAAc,EAAE,CAAC;oBACzB,UAAU,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YACD,OAAO,cAAc,GAAG,UAAU,CAAC;QACrC,CAAC;QAED,UAAU,CAAC,gBAAwB;YACjC,qCAAqC;YACrC,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,IAAI,iBAAiB,GAAG,CAAC,GAAG,GAAG,UAAU,CAAC,IAAI,gBAAgB,EAAE,CAAC;oBAC/D,UAAU,EAAE,CAAC;oBACb,iBAAiB,GAAG,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,OAAO,gBAAgB,GAAG,UAAU,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Entity Resolver
3
+ * Merges, deduplicates, and resolves overlapping entity detections
4
+ */
5
+ import { SpanMatch, AnonymizationPolicy } from '../types/index.js';
6
+ /**
7
+ * Resolution strategy for overlapping entities
8
+ */
9
+ export declare enum OverlapStrategy {
10
+ /** Regex matches always win over NER */
11
+ REGEX_PRIORITY = "REGEX_PRIORITY",
12
+ /** Longer span wins */
13
+ LONGER_SPAN = "LONGER_SPAN",
14
+ /** Higher confidence wins */
15
+ HIGHER_CONFIDENCE = "HIGHER_CONFIDENCE",
16
+ /** Use type priority from policy */
17
+ TYPE_PRIORITY = "TYPE_PRIORITY"
18
+ }
19
+ /**
20
+ * Entity resolver configuration
21
+ */
22
+ export interface ResolverConfig {
23
+ /** Primary strategy for overlap resolution */
24
+ overlapStrategy: OverlapStrategy;
25
+ /** Whether regex matches always take precedence */
26
+ regexPriority: boolean;
27
+ /** Minimum confidence to keep an entity */
28
+ minConfidence: number;
29
+ }
30
+ /**
31
+ * Default resolver configuration
32
+ */
33
+ export declare const DEFAULT_RESOLVER_CONFIG: ResolverConfig;
34
+ /**
35
+ * Resolves and merges entity detections from regex and NER
36
+ */
37
+ export declare function resolveEntities(regexMatches: SpanMatch[], nerMatches: SpanMatch[], policy: AnonymizationPolicy, originalText: string, config?: Partial<ResolverConfig>): SpanMatch[];
38
+ /**
39
+ * Creates protected spans from regex matches
40
+ * Used to mask regex matches from NER to avoid double-detection
41
+ */
42
+ export declare function createProtectedSpans(regexMatches: SpanMatch[]): Array<{
43
+ start: number;
44
+ end: number;
45
+ }>;
46
+ /**
47
+ * Checks if a span overlaps with any protected span
48
+ */
49
+ export declare function isInProtectedSpan(span: {
50
+ start: number;
51
+ end: number;
52
+ }, protectedSpans: Array<{
53
+ start: number;
54
+ end: number;
55
+ }>): boolean;
56
+ //# sourceMappingURL=resolver.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/pipeline/resolver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAEL,SAAS,EAET,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAG3B;;GAEG;AACH,oBAAY,eAAe;IACzB,wCAAwC;IACxC,cAAc,mBAAmB;IACjC,uBAAuB;IACvB,WAAW,gBAAgB;IAC3B,6BAA6B;IAC7B,iBAAiB,sBAAsB;IACvC,oCAAoC;IACpC,aAAa,kBAAkB;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8CAA8C;IAC9C,eAAe,EAAE,eAAe,CAAC;IACjC,mDAAmD;IACnD,aAAa,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,cAIrC,CAAC;AAEF;;GAEG;AACH,wBAAgB,eAAe,CAC7B,YAAY,EAAE,SAAS,EAAE,EACzB,UAAU,EAAE,SAAS,EAAE,EACvB,MAAM,EAAE,mBAAmB,EAC3B,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GACnC,SAAS,EAAE,CAyBb;AAkOD;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,SAAS,EAAE,GACxB,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAEvC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACpC,cAAc,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,GACpD,OAAO,CAET"}
@@ -0,0 +1,239 @@
1
+ /**
2
+ * Entity Resolver
3
+ * Merges, deduplicates, and resolves overlapping entity detections
4
+ */
5
+ import { PIIType, DetectionSource, DEFAULT_TYPE_PRIORITY, } from '../types/index.js';
6
+ import { spansOverlap, spanLength, sortSpansByPosition } from '../utils/offsets.js';
7
+ /**
8
+ * Resolution strategy for overlapping entities
9
+ */
10
+ export var OverlapStrategy;
11
+ (function (OverlapStrategy) {
12
+ /** Regex matches always win over NER */
13
+ OverlapStrategy["REGEX_PRIORITY"] = "REGEX_PRIORITY";
14
+ /** Longer span wins */
15
+ OverlapStrategy["LONGER_SPAN"] = "LONGER_SPAN";
16
+ /** Higher confidence wins */
17
+ OverlapStrategy["HIGHER_CONFIDENCE"] = "HIGHER_CONFIDENCE";
18
+ /** Use type priority from policy */
19
+ OverlapStrategy["TYPE_PRIORITY"] = "TYPE_PRIORITY";
20
+ })(OverlapStrategy || (OverlapStrategy = {}));
21
+ /**
22
+ * Default resolver configuration
23
+ */
24
+ export const DEFAULT_RESOLVER_CONFIG = {
25
+ overlapStrategy: OverlapStrategy.REGEX_PRIORITY,
26
+ regexPriority: true,
27
+ minConfidence: 0.5,
28
+ };
29
+ /**
30
+ * Resolves and merges entity detections from regex and NER
31
+ */
32
+ export function resolveEntities(regexMatches, nerMatches, policy, originalText, config = {}) {
33
+ const resolverConfig = { ...DEFAULT_RESOLVER_CONFIG, ...config };
34
+ // Step 1: Filter by enabled types and confidence thresholds
35
+ const filteredRegex = filterByPolicy(regexMatches, policy);
36
+ const filteredNER = filterByPolicy(nerMatches, policy);
37
+ // Step 2: Apply allowlist filtering
38
+ const allowlistFilteredRegex = applyAllowlist(filteredRegex, policy, originalText);
39
+ const allowlistFilteredNER = applyAllowlist(filteredNER, policy, originalText);
40
+ // Step 3: Combine all matches
41
+ const allMatches = [...allowlistFilteredRegex, ...allowlistFilteredNER];
42
+ // Step 4: Remove overlaps based on strategy
43
+ const resolved = removeOverlaps(allMatches, policy, resolverConfig);
44
+ // Step 5: Apply denylist patterns (force include)
45
+ const withDenylist = applyDenylist(resolved, policy, originalText);
46
+ // Step 6: Final deduplication
47
+ const deduplicated = deduplicateExact(withDenylist);
48
+ // Step 7: Sort by position
49
+ return sortSpansByPosition(deduplicated);
50
+ }
51
+ /**
52
+ * Filters matches by policy (enabled types and confidence thresholds)
53
+ */
54
+ function filterByPolicy(matches, policy) {
55
+ return matches.filter((match) => {
56
+ // Check if type is enabled
57
+ if (!policy.enabledTypes.has(match.type)) {
58
+ return false;
59
+ }
60
+ // Check confidence threshold
61
+ const threshold = policy.confidenceThresholds.get(match.type) ?? 0.5;
62
+ if (match.confidence < threshold) {
63
+ return false;
64
+ }
65
+ return true;
66
+ });
67
+ }
68
+ /**
69
+ * Filters out matches that are in the allowlist (known non-PII terms)
70
+ */
71
+ function applyAllowlist(matches, policy, _originalText) {
72
+ if (policy.allowlistTerms.size === 0) {
73
+ return matches;
74
+ }
75
+ return matches.filter((match) => {
76
+ const matchText = match.text.toLowerCase().trim();
77
+ return !policy.allowlistTerms.has(matchText);
78
+ });
79
+ }
80
+ /**
81
+ * Adds matches from denylist patterns (patterns that must always be PII)
82
+ */
83
+ function applyDenylist(matches, policy, originalText) {
84
+ if (policy.denylistPatterns.length === 0) {
85
+ return matches;
86
+ }
87
+ const denylistMatches = [];
88
+ for (const pattern of policy.denylistPatterns) {
89
+ const globalPattern = pattern.global
90
+ ? pattern
91
+ : new RegExp(pattern.source, pattern.flags + 'g');
92
+ for (const match of originalText.matchAll(globalPattern)) {
93
+ if (match.index === undefined)
94
+ continue;
95
+ // Check if this is already covered by existing matches
96
+ const matchIndex = match.index;
97
+ const alreadyCovered = matches.some((existing) => existing.start <= matchIndex &&
98
+ existing.end >= matchIndex + match[0].length);
99
+ if (!alreadyCovered) {
100
+ denylistMatches.push({
101
+ type: PIIType.EMAIL, // Default type for denylist; could be configurable
102
+ start: match.index,
103
+ end: match.index + match[0].length,
104
+ confidence: 1.0,
105
+ source: DetectionSource.REGEX,
106
+ text: match[0],
107
+ });
108
+ }
109
+ }
110
+ }
111
+ return [...matches, ...denylistMatches];
112
+ }
113
+ /**
114
+ * Removes overlapping spans based on resolution strategy
115
+ */
116
+ function removeOverlaps(matches, policy, config) {
117
+ if (matches.length <= 1) {
118
+ return matches;
119
+ }
120
+ // Sort by start position
121
+ const sorted = sortSpansByPosition(matches);
122
+ const result = [];
123
+ for (const match of sorted) {
124
+ // Find overlapping matches in result
125
+ const overlappingIdx = result.findIndex((existing) => spansOverlap(match, existing));
126
+ if (overlappingIdx === -1) {
127
+ // No overlap, add directly
128
+ result.push(match);
129
+ }
130
+ else {
131
+ // Has overlap, resolve
132
+ const existing = result[overlappingIdx];
133
+ const winner = resolveOverlap(existing, match, policy, config);
134
+ if (winner === match) {
135
+ // New match wins, replace existing
136
+ result[overlappingIdx] = match;
137
+ }
138
+ // Otherwise keep existing (do nothing)
139
+ }
140
+ }
141
+ return result;
142
+ }
143
+ /**
144
+ * Resolves overlap between two spans
145
+ * Returns the winner
146
+ */
147
+ function resolveOverlap(a, b, policy, config) {
148
+ // Rule 1: Regex always beats NER if configured
149
+ if (config.regexPriority) {
150
+ if (a.source === DetectionSource.REGEX && b.source !== DetectionSource.REGEX) {
151
+ return a;
152
+ }
153
+ if (b.source === DetectionSource.REGEX && a.source !== DetectionSource.REGEX) {
154
+ return b;
155
+ }
156
+ }
157
+ // Rule 2: Apply overlap strategy
158
+ switch (config.overlapStrategy) {
159
+ case OverlapStrategy.LONGER_SPAN: {
160
+ const lenA = spanLength(a);
161
+ const lenB = spanLength(b);
162
+ if (lenA !== lenB) {
163
+ return lenA > lenB ? a : b;
164
+ }
165
+ break;
166
+ }
167
+ case OverlapStrategy.HIGHER_CONFIDENCE: {
168
+ if (a.confidence !== b.confidence) {
169
+ return a.confidence > b.confidence ? a : b;
170
+ }
171
+ break;
172
+ }
173
+ case OverlapStrategy.TYPE_PRIORITY: {
174
+ const priorityA = getTypePriority(a.type, policy);
175
+ const priorityB = getTypePriority(b.type, policy);
176
+ if (priorityA !== priorityB) {
177
+ return priorityA > priorityB ? a : b;
178
+ }
179
+ break;
180
+ }
181
+ case OverlapStrategy.REGEX_PRIORITY:
182
+ default:
183
+ // Already handled above
184
+ break;
185
+ }
186
+ // Tiebreakers: longer span > higher confidence > type priority
187
+ const lenA = spanLength(a);
188
+ const lenB = spanLength(b);
189
+ if (lenA !== lenB) {
190
+ return lenA > lenB ? a : b;
191
+ }
192
+ if (a.confidence !== b.confidence) {
193
+ return a.confidence > b.confidence ? a : b;
194
+ }
195
+ const priorityA = getTypePriority(a.type, policy);
196
+ const priorityB = getTypePriority(b.type, policy);
197
+ if (priorityA !== priorityB) {
198
+ return priorityA > priorityB ? a : b;
199
+ }
200
+ // Final tiebreaker: keep first one
201
+ return a;
202
+ }
203
+ /**
204
+ * Gets type priority from policy (higher = more important)
205
+ */
206
+ function getTypePriority(type, policy) {
207
+ const priorityList = policy.typePriority.length > 0 ? policy.typePriority : [...DEFAULT_TYPE_PRIORITY];
208
+ const index = priorityList.indexOf(type);
209
+ return index >= 0 ? index : -1;
210
+ }
211
+ /**
212
+ * Removes exact duplicate spans
213
+ */
214
+ function deduplicateExact(matches) {
215
+ const seen = new Set();
216
+ const result = [];
217
+ for (const match of matches) {
218
+ const key = `${match.start}:${match.end}:${match.type}`;
219
+ if (!seen.has(key)) {
220
+ seen.add(key);
221
+ result.push(match);
222
+ }
223
+ }
224
+ return result;
225
+ }
226
+ /**
227
+ * Creates protected spans from regex matches
228
+ * Used to mask regex matches from NER to avoid double-detection
229
+ */
230
+ export function createProtectedSpans(regexMatches) {
231
+ return regexMatches.map(({ start, end }) => ({ start, end }));
232
+ }
233
+ /**
234
+ * Checks if a span overlaps with any protected span
235
+ */
236
+ export function isInProtectedSpan(span, protectedSpans) {
237
+ return protectedSpans.some((protected_) => spansOverlap(span, protected_));
238
+ }
239
+ //# sourceMappingURL=resolver.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"resolver.js","sourceRoot":"","sources":["../../src/pipeline/resolver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,OAAO,EAEP,eAAe,EAEf,qBAAqB,GACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF;;GAEG;AACH,MAAM,CAAN,IAAY,eASX;AATD,WAAY,eAAe;IACzB,wCAAwC;IACxC,oDAAiC,CAAA;IACjC,uBAAuB;IACvB,8CAA2B,CAAA;IAC3B,6BAA6B;IAC7B,0DAAuC,CAAA;IACvC,oCAAoC;IACpC,kDAA+B,CAAA;AACjC,CAAC,EATW,eAAe,KAAf,eAAe,QAS1B;AAcD;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAmB;IACrD,eAAe,EAAE,eAAe,CAAC,cAAc;IAC/C,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,YAAyB,EACzB,UAAuB,EACvB,MAA2B,EAC3B,YAAoB,EACpB,SAAkC,EAAE;IAEpC,MAAM,cAAc,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAAC;IAEjE,4DAA4D;IAC5D,MAAM,aAAa,GAAG,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAEvD,oCAAoC;IACpC,MAAM,sBAAsB,GAAG,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IACnF,MAAM,oBAAoB,GAAG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAE/E,8BAA8B;IAC9B,MAAM,UAAU,GAAG,CAAC,GAAG,sBAAsB,EAAE,GAAG,oBAAoB,CAAC,CAAC;IAExE,4CAA4C;IAC5C,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC;IAEpE,kDAAkD;IAClD,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnE,8BAA8B;IAC9B,MAAM,YAAY,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAEpD,2BAA2B;IAC3B,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAoB,EAAE,MAA2B;IACvE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,2BAA2B;QAC3B,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,6BAA6B;QAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QACrE,IAAI,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,OAAoB,EACpB,MAA2B,EAC3B,aAAqB;IAErB,IAAI,MAAM,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CACpB,OAAoB,EACpB,MAA2B,EAC3B,YAAoB;IAEpB,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,eAAe,GAAgB,EAAE,CAAC;IAExC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM;YAClC,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;QAEpD,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAS;YAExC,uDAAuD;YACvD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;YAC/B,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CACjC,CAAC,QAAQ,EAAE,EAAE,CACX,QAAQ,CAAC,KAAK,IAAI,UAAU;gBAC5B,QAAQ,CAAC,GAAG,IAAI,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAC/C,CAAC;YAEF,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,eAAe,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,mDAAmD;oBACxE,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;oBAClC,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,eAAe,CAAC,KAAK;oBAC7B,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,EAAE,GAAG,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,OAAoB,EACpB,MAA2B,EAC3B,MAAsB;IAEtB,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,yBAAyB;IACzB,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,qCAAqC;QACrC,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAErF,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1B,2BAA2B;YAC3B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,uBAAuB;YACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAE,CAAC;YACzC,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAE/D,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBACrB,mCAAmC;gBACnC,MAAM,CAAC,cAAc,CAAC,GAAG,KAAK,CAAC;YACjC,CAAC;YACD,uCAAuC;QACzC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CACrB,CAAY,EACZ,CAAY,EACZ,MAA2B,EAC3B,MAAsB;IAEtB,+CAA+C;IAC/C,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7E,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7E,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/B,KAAK,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;YACjC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,OAAO,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;gBAClC,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,eAAe,CAAC,cAAc,CAAC;QACpC;YACE,wBAAwB;YACxB,MAAM;IACV,CAAC;IAED,+DAA+D;IAC/D,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;QAClC,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,mCAAmC;IACnC,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAa,EAAE,MAA2B;IACjE,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAoB;IAC5C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,YAAyB;IAEzB,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAoC,EACpC,cAAqD;IAErD,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;AAC7E,CAAC"}
@@ -0,0 +1,171 @@
1
+ /**
2
+ * Semantic Data Loader
3
+ * Handles automatic downloading, caching, and parsing of semantic enrichment data.
4
+ * Browser-compatible using storage abstraction layer
5
+ *
6
+ * Data sources:
7
+ * - nam_dict.txt: Name-gender mappings from gender-guesser (~40K names)
8
+ * - cities15000.txt: GeoNames cities with population > 15,000 (~25K cities)
9
+ * - countryInfo.txt: Country names and codes (~250 countries)
10
+ * - admin1CodesASCII.txt: First-level admin divisions (~4K regions)
11
+ *
12
+ * Data is cached using the storage abstraction layer:
13
+ * - Node.js: Platform-specific cache directories
14
+ * - Browser: IndexedDB
15
+ */
16
+ /**
17
+ * Loaded name-gender data
18
+ */
19
+ interface NameEntry {
20
+ gender: string;
21
+ localeOverrides?: Record<string, string>;
22
+ }
23
+ /**
24
+ * Loaded location data
25
+ */
26
+ interface CityEntry {
27
+ country: string;
28
+ population: number;
29
+ }
30
+ interface RegionEntry {
31
+ country: string;
32
+ name: string;
33
+ }
34
+ /**
35
+ * Semantic data store
36
+ */
37
+ interface SemanticData {
38
+ names: Map<string, NameEntry>;
39
+ cities: Map<string, CityEntry>;
40
+ countries: Map<string, string>;
41
+ regions: Map<string, RegionEntry>;
42
+ loaded: boolean;
43
+ }
44
+ /**
45
+ * Gets the cache directory for semantic data
46
+ * Uses platform-specific cache location (or virtual path in browser)
47
+ */
48
+ export declare function getSemanticDataCacheDir(): Promise<string>;
49
+ /**
50
+ * Gets the path to the data directory (alias for backwards compatibility)
51
+ */
52
+ export declare function getDataDirectory(): Promise<string>;
53
+ /**
54
+ * Semantic data file info
55
+ */
56
+ export interface SemanticDataFileInfo {
57
+ /** Filename */
58
+ filename: string;
59
+ /** Download URL */
60
+ url: string;
61
+ /** Whether file is required */
62
+ required: boolean;
63
+ /** Description */
64
+ description: string;
65
+ /** Approximate size */
66
+ size: string;
67
+ }
68
+ /**
69
+ * Registry of semantic data files and their download URLs
70
+ */
71
+ export declare const SEMANTIC_DATA_FILES: SemanticDataFileInfo[];
72
+ /**
73
+ * Progress callback for downloads
74
+ */
75
+ export type SemanticDownloadProgressCallback = (progress: {
76
+ file: string;
77
+ bytesDownloaded: number;
78
+ totalBytes: number | null;
79
+ percent: number | null;
80
+ }) => void;
81
+ /**
82
+ * Checks if semantic data is already downloaded
83
+ */
84
+ export declare function isSemanticDataDownloaded(): Promise<boolean>;
85
+ /**
86
+ * Checks if the semantic data files are available
87
+ * Note: This is now async to work with browser storage
88
+ */
89
+ export declare function isSemanticDataAvailable(): Promise<boolean>;
90
+ /**
91
+ * Downloads all semantic data files
92
+ */
93
+ export declare function downloadSemanticData(onProgress?: SemanticDownloadProgressCallback, onStatus?: (status: string) => void): Promise<string>;
94
+ /**
95
+ * Ensures semantic data is available, downloading if needed
96
+ */
97
+ export declare function ensureSemanticData(options?: {
98
+ autoDownload?: boolean;
99
+ onProgress?: SemanticDownloadProgressCallback;
100
+ onStatus?: (status: string) => void;
101
+ }): Promise<string>;
102
+ /**
103
+ * Clears cached semantic data
104
+ */
105
+ export declare function clearSemanticDataCache(): Promise<void>;
106
+ /**
107
+ * Gets info about semantic data files
108
+ */
109
+ export declare function getSemanticDataInfo(): Promise<{
110
+ files: SemanticDataFileInfo[];
111
+ cacheDir: string;
112
+ totalSize: string;
113
+ }>;
114
+ /**
115
+ * Initializes semantic data (downloads if needed, then loads)
116
+ */
117
+ export declare function initializeSemanticData(options?: {
118
+ autoDownload?: boolean;
119
+ onProgress?: SemanticDownloadProgressCallback;
120
+ onStatus?: (status: string) => void;
121
+ }): Promise<void>;
122
+ /**
123
+ * Loads semantic data from cached files
124
+ * @throws Error if required data files are not available
125
+ */
126
+ export declare function loadSemanticData(): Promise<SemanticData>;
127
+ /**
128
+ * Gets the loaded semantic data (loads if not already loaded)
129
+ * Note: This is now async to work with browser storage
130
+ */
131
+ export declare function getSemanticData(): Promise<SemanticData>;
132
+ /**
133
+ * Gets the loaded semantic data synchronously
134
+ * Returns null if data is not yet loaded
135
+ * Use this for performance-critical code paths after initial loading
136
+ */
137
+ export declare function getSemanticDataSync(): SemanticData | null;
138
+ /**
139
+ * Clears the loaded semantic data (useful for testing)
140
+ */
141
+ export declare function clearSemanticData(): void;
142
+ /**
143
+ * Looks up gender for a name
144
+ */
145
+ export declare function lookupGender(name: string, locale?: string): Promise<string | undefined>;
146
+ /**
147
+ * Continents and large named geographic areas (larger than any single country).
148
+ * Checked before countries so that "Europe", "South America", etc. are not
149
+ * misclassified when a country happens to share a substring.
150
+ */
151
+ export declare const MACRO_REGIONS: Set<string>;
152
+ /**
153
+ * Looks up location type (city, country, or region)
154
+ * Priority: macro-region > country > major city (pop > 500K) > region > other cities
155
+ */
156
+ export declare function lookupLocationType(location: string): Promise<{
157
+ type: "city" | "country" | "region" | "macro-region";
158
+ countryCode?: string;
159
+ } | undefined>;
160
+ /**
161
+ * Gets statistics about loaded data
162
+ */
163
+ export declare function getDataStats(): {
164
+ names: number;
165
+ cities: number;
166
+ countries: number;
167
+ regions: number;
168
+ loaded: boolean;
169
+ };
170
+ export {};
171
+ //# sourceMappingURL=semantic-data-loader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"semantic-data-loader.d.ts","sourceRoot":"","sources":["../../src/pipeline/semantic-data-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA6EH;;GAEG;AACH,UAAU,SAAS;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC9B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClC,MAAM,EAAE,OAAO,CAAC;CACjB;AAsBD;;;GAGG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,MAAM,CAAC,CAG/D;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAExD;AAMD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,eAAe;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,kBAAkB;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;AAQD;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,oBAAoB,EA6BrD,CAAC;AAMF;;GAEG;AACH,MAAM,MAAM,gCAAgC,GAAG,CAAC,QAAQ,EAAE;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,KAAK,IAAI,CAAC;AAEX;;GAEG;AACH,wBAAsB,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC,CAgBjE;AAED;;;GAGG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,OAAO,CAAC,CAEhE;AAuFD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,UAAU,CAAC,EAAE,gCAAgC,EAC7C,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC,CA+BjB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,gCAAgC,CAAC;IAC9C,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC,MAAM,CAAC,CAyBjB;AAED;;GAEG;AACH,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC,CAM5D;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC;IACnD,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC,CAOD;AAqPD;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,gCAAgC,CAAC;IAC9C,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,YAAY,CAAC,CAqD9D;AAED;;;GAGG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC,CAK7D;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,YAAY,GAAG,IAAI,CAKzD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAiB7B;AAOD;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,CAYpC,CAAC;AAEH;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,GACf,OAAO,CACR;IAAE,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,GAAG,cAAc,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAC3F,CAiCA;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB,CAYA"}