opencode-rehydar 0.14.1 → 0.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +54 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +74 -0
- package/dist/browser.js.map +1 -0
- package/dist/core/anonymizer.d.ts +309 -0
- package/dist/core/anonymizer.d.ts.map +1 -0
- package/dist/core/anonymizer.js +555 -0
- package/dist/core/anonymizer.js.map +1 -0
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +6 -0
- package/dist/core/index.js.map +1 -0
- package/dist/crypto/index.d.ts +6 -0
- package/dist/crypto/index.d.ts.map +1 -0
- package/dist/crypto/index.js +6 -0
- package/dist/crypto/index.js.map +1 -0
- package/dist/crypto/pii-map-crypto.d.ts +114 -0
- package/dist/crypto/pii-map-crypto.d.ts.map +1 -0
- package/dist/crypto/pii-map-crypto.js +228 -0
- package/dist/crypto/pii-map-crypto.js.map +1 -0
- package/dist/index.d.ts +53 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +72 -0
- package/dist/index.js.map +1 -0
- package/dist/ner/bio-decoder.d.ts +64 -0
- package/dist/ner/bio-decoder.d.ts.map +1 -0
- package/dist/ner/bio-decoder.js +218 -0
- package/dist/ner/bio-decoder.js.map +1 -0
- package/dist/ner/index.d.ts +12 -0
- package/dist/ner/index.d.ts.map +1 -0
- package/dist/ner/index.js +12 -0
- package/dist/ner/index.js.map +1 -0
- package/dist/ner/inference-server-client.d.ts +81 -0
- package/dist/ner/inference-server-client.d.ts.map +1 -0
- package/dist/ner/inference-server-client.js +120 -0
- package/dist/ner/inference-server-client.js.map +1 -0
- package/dist/ner/inference-server-ner-model.d.ts +43 -0
- package/dist/ner/inference-server-ner-model.d.ts.map +1 -0
- package/dist/ner/inference-server-ner-model.js +119 -0
- package/dist/ner/inference-server-ner-model.js.map +1 -0
- package/dist/ner/model-manager.d.ts +111 -0
- package/dist/ner/model-manager.d.ts.map +1 -0
- package/dist/ner/model-manager.js +334 -0
- package/dist/ner/model-manager.js.map +1 -0
- package/dist/ner/ner-model.d.ts +172 -0
- package/dist/ner/ner-model.d.ts.map +1 -0
- package/dist/ner/ner-model.js +514 -0
- package/dist/ner/ner-model.js.map +1 -0
- package/dist/ner/onnx-runtime.browser.d.ts +46 -0
- package/dist/ner/onnx-runtime.browser.d.ts.map +1 -0
- package/dist/ner/onnx-runtime.browser.js +90 -0
- package/dist/ner/onnx-runtime.browser.js.map +1 -0
- package/dist/ner/onnx-runtime.d.ts +72 -0
- package/dist/ner/onnx-runtime.d.ts.map +1 -0
- package/dist/ner/onnx-runtime.js +132 -0
- package/dist/ner/onnx-runtime.js.map +1 -0
- package/dist/ner/token-window.d.ts +19 -0
- package/dist/ner/token-window.d.ts.map +1 -0
- package/dist/ner/token-window.js +101 -0
- package/dist/ner/token-window.js.map +1 -0
- package/dist/ner/tokenizer.d.ts +142 -0
- package/dist/ner/tokenizer.d.ts.map +1 -0
- package/dist/ner/tokenizer.js +362 -0
- package/dist/ner/tokenizer.js.map +1 -0
- package/dist/opencode-plugin/git-identity.d.ts +3 -0
- package/dist/opencode-plugin/git-identity.d.ts.map +1 -0
- package/dist/opencode-plugin/git-identity.js +37 -0
- package/dist/opencode-plugin/git-identity.js.map +1 -0
- package/dist/opencode-plugin/github-username.d.ts +3 -0
- package/dist/opencode-plugin/github-username.d.ts.map +1 -0
- package/dist/opencode-plugin/github-username.js +41 -0
- package/dist/opencode-plugin/github-username.js.map +1 -0
- package/dist/opencode-plugin/identity-recognizer.d.ts +5 -0
- package/dist/opencode-plugin/identity-recognizer.d.ts.map +1 -0
- package/dist/opencode-plugin/identity-recognizer.js +21 -0
- package/dist/opencode-plugin/identity-recognizer.js.map +1 -0
- package/dist/opencode-plugin/index.d.ts +7 -0
- package/dist/opencode-plugin/index.d.ts.map +1 -0
- package/dist/opencode-plugin/index.js +6 -0
- package/dist/opencode-plugin/index.js.map +1 -0
- package/dist/opencode-plugin/plugin.d.ts +39 -0
- package/dist/opencode-plugin/plugin.d.ts.map +1 -0
- package/dist/opencode-plugin/plugin.js +294 -0
- package/dist/opencode-plugin/plugin.js.map +1 -0
- package/dist/opencode-plugin/types.d.ts +33 -0
- package/dist/opencode-plugin/types.d.ts.map +1 -0
- package/dist/opencode-plugin/types.js +5 -0
- package/dist/opencode-plugin/types.js.map +1 -0
- package/dist/opencode-plugin/v2/anon-state.d.ts +12 -0
- package/dist/opencode-plugin/v2/anon-state.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/anon-state.js +14 -0
- package/dist/opencode-plugin/v2/anon-state.js.map +1 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.d.ts +13 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.js +33 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.js.map +1 -0
- package/dist/opencode-plugin/v2/index.d.ts +17 -0
- package/dist/opencode-plugin/v2/index.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/index.js +17 -0
- package/dist/opencode-plugin/v2/index.js.map +1 -0
- package/dist/opencode-plugin/v2/logger.d.ts +24 -0
- package/dist/opencode-plugin/v2/logger.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/logger.js +73 -0
- package/dist/opencode-plugin/v2/logger.js.map +1 -0
- package/dist/opencode-plugin/v2/options.d.ts +63 -0
- package/dist/opencode-plugin/v2/options.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/options.js +183 -0
- package/dist/opencode-plugin/v2/options.js.map +1 -0
- package/dist/opencode-plugin/v2/register-hooks.d.ts +43 -0
- package/dist/opencode-plugin/v2/register-hooks.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/register-hooks.js +160 -0
- package/dist/opencode-plugin/v2/register-hooks.js.map +1 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.d.ts +101 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.js +147 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.js.map +1 -0
- package/dist/opencode-plugin/v2/response-rehydrate.d.ts +78 -0
- package/dist/opencode-plugin/v2/response-rehydrate.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/response-rehydrate.js +210 -0
- package/dist/opencode-plugin/v2/response-rehydrate.js.map +1 -0
- package/dist/opencode-plugin/v2/scrub-messages.d.ts +39 -0
- package/dist/opencode-plugin/v2/scrub-messages.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/scrub-messages.js +260 -0
- package/dist/opencode-plugin/v2/scrub-messages.js.map +1 -0
- package/dist/opencode-plugin/v2/sessions.d.ts +17 -0
- package/dist/opencode-plugin/v2/sessions.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/sessions.js +23 -0
- package/dist/opencode-plugin/v2/sessions.js.map +1 -0
- package/dist/opencode-plugin/v2/sse.d.ts +101 -0
- package/dist/opencode-plugin/v2/sse.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/sse.js +235 -0
- package/dist/opencode-plugin/v2/sse.js.map +1 -0
- package/dist/opencode-plugin/v2/types.d.ts +188 -0
- package/dist/opencode-plugin/v2/types.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/types.js +10 -0
- package/dist/opencode-plugin/v2/types.js.map +1 -0
- package/dist/opencode-plugin/vcs-command.d.ts +3 -0
- package/dist/opencode-plugin/vcs-command.d.ts.map +1 -0
- package/dist/opencode-plugin/vcs-command.js +34 -0
- package/dist/opencode-plugin/vcs-command.js.map +1 -0
- package/dist/pipeline/index.d.ts +12 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/pipeline/index.js +12 -0
- package/dist/pipeline/index.js.map +1 -0
- package/dist/pipeline/prenormalize.d.ts +48 -0
- package/dist/pipeline/prenormalize.d.ts.map +1 -0
- package/dist/pipeline/prenormalize.js +94 -0
- package/dist/pipeline/prenormalize.js.map +1 -0
- package/dist/pipeline/resolver.d.ts +56 -0
- package/dist/pipeline/resolver.d.ts.map +1 -0
- package/dist/pipeline/resolver.js +239 -0
- package/dist/pipeline/resolver.js.map +1 -0
- package/dist/pipeline/semantic-data-loader.d.ts +171 -0
- package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
- package/dist/pipeline/semantic-data-loader.js +677 -0
- package/dist/pipeline/semantic-data-loader.js.map +1 -0
- package/dist/pipeline/semantic-enricher.d.ts +112 -0
- package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
- package/dist/pipeline/semantic-enricher.js +322 -0
- package/dist/pipeline/semantic-enricher.js.map +1 -0
- package/dist/pipeline/tagger.d.ts +121 -0
- package/dist/pipeline/tagger.d.ts.map +1 -0
- package/dist/pipeline/tagger.js +545 -0
- package/dist/pipeline/tagger.js.map +1 -0
- package/dist/pipeline/title-extractor.d.ts +79 -0
- package/dist/pipeline/title-extractor.d.ts.map +1 -0
- package/dist/pipeline/title-extractor.js +801 -0
- package/dist/pipeline/title-extractor.js.map +1 -0
- package/dist/pipeline/validator.d.ts +65 -0
- package/dist/pipeline/validator.d.ts.map +1 -0
- package/dist/pipeline/validator.js +292 -0
- package/dist/pipeline/validator.js.map +1 -0
- package/dist/proxy/index.d.ts +13 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +12 -0
- package/dist/proxy/index.js.map +1 -0
- package/dist/proxy/providers/anthropic.d.ts +28 -0
- package/dist/proxy/providers/anthropic.d.ts.map +1 -0
- package/dist/proxy/providers/anthropic.js +265 -0
- package/dist/proxy/providers/anthropic.js.map +1 -0
- package/dist/proxy/providers/index.d.ts +19 -0
- package/dist/proxy/providers/index.d.ts.map +1 -0
- package/dist/proxy/providers/index.js +41 -0
- package/dist/proxy/providers/index.js.map +1 -0
- package/dist/proxy/providers/openai.d.ts +27 -0
- package/dist/proxy/providers/openai.d.ts.map +1 -0
- package/dist/proxy/providers/openai.js +203 -0
- package/dist/proxy/providers/openai.js.map +1 -0
- package/dist/proxy/providers/streaming-tool-loop.d.ts +12 -0
- package/dist/proxy/providers/streaming-tool-loop.d.ts.map +1 -0
- package/dist/proxy/providers/streaming-tool-loop.js +223 -0
- package/dist/proxy/providers/streaming-tool-loop.js.map +1 -0
- package/dist/proxy/providers/types.d.ts +109 -0
- package/dist/proxy/providers/types.d.ts.map +1 -0
- package/dist/proxy/providers/types.js +2 -0
- package/dist/proxy/providers/types.js.map +1 -0
- package/dist/proxy/proxy-server.d.ts +61 -0
- package/dist/proxy/proxy-server.d.ts.map +1 -0
- package/dist/proxy/proxy-server.js +146 -0
- package/dist/proxy/proxy-server.js.map +1 -0
- package/dist/proxy/rehydra-fetch.d.ts +35 -0
- package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
- package/dist/proxy/rehydra-fetch.js +631 -0
- package/dist/proxy/rehydra-fetch.js.map +1 -0
- package/dist/proxy/rehydra-proxy.d.ts +40 -0
- package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
- package/dist/proxy/rehydra-proxy.js +150 -0
- package/dist/proxy/rehydra-proxy.js.map +1 -0
- package/dist/proxy/sse-parser.d.ts +59 -0
- package/dist/proxy/sse-parser.d.ts.map +1 -0
- package/dist/proxy/sse-parser.js +112 -0
- package/dist/proxy/sse-parser.js.map +1 -0
- package/dist/proxy/system-instruction.d.ts +19 -0
- package/dist/proxy/system-instruction.d.ts.map +1 -0
- package/dist/proxy/system-instruction.js +33 -0
- package/dist/proxy/system-instruction.js.map +1 -0
- package/dist/proxy/tool-loop-stream.d.ts +7 -0
- package/dist/proxy/tool-loop-stream.d.ts.map +1 -0
- package/dist/proxy/tool-loop-stream.js +65 -0
- package/dist/proxy/tool-loop-stream.js.map +1 -0
- package/dist/proxy/types.d.ts +138 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +5 -0
- package/dist/proxy/types.js.map +1 -0
- package/dist/proxy/wrap-client.d.ts +47 -0
- package/dist/proxy/wrap-client.d.ts.map +1 -0
- package/dist/proxy/wrap-client.js +70 -0
- package/dist/proxy/wrap-client.js.map +1 -0
- package/dist/recognizers/address.d.ts +2 -0
- package/dist/recognizers/address.d.ts.map +1 -0
- package/dist/recognizers/address.js +23 -0
- package/dist/recognizers/address.js.map +1 -0
- package/dist/recognizers/base.d.ts +78 -0
- package/dist/recognizers/base.d.ts.map +1 -0
- package/dist/recognizers/base.js +100 -0
- package/dist/recognizers/base.js.map +1 -0
- package/dist/recognizers/bic-swift.d.ts +10 -0
- package/dist/recognizers/bic-swift.d.ts.map +1 -0
- package/dist/recognizers/bic-swift.js +107 -0
- package/dist/recognizers/bic-swift.js.map +1 -0
- package/dist/recognizers/credit-card.d.ts +32 -0
- package/dist/recognizers/credit-card.d.ts.map +1 -0
- package/dist/recognizers/credit-card.js +160 -0
- package/dist/recognizers/credit-card.js.map +1 -0
- package/dist/recognizers/custom-id.d.ts +28 -0
- package/dist/recognizers/custom-id.d.ts.map +1 -0
- package/dist/recognizers/custom-id.js +116 -0
- package/dist/recognizers/custom-id.js.map +1 -0
- package/dist/recognizers/date.d.ts +10 -0
- package/dist/recognizers/date.d.ts.map +1 -0
- package/dist/recognizers/date.js +228 -0
- package/dist/recognizers/date.js.map +1 -0
- package/dist/recognizers/email.d.ts +10 -0
- package/dist/recognizers/email.d.ts.map +1 -0
- package/dist/recognizers/email.js +75 -0
- package/dist/recognizers/email.js.map +1 -0
- package/dist/recognizers/iban.d.ts +14 -0
- package/dist/recognizers/iban.d.ts.map +1 -0
- package/dist/recognizers/iban.js +67 -0
- package/dist/recognizers/iban.js.map +1 -0
- package/dist/recognizers/index.d.ts +25 -0
- package/dist/recognizers/index.d.ts.map +1 -0
- package/dist/recognizers/index.js +53 -0
- package/dist/recognizers/index.js.map +1 -0
- package/dist/recognizers/ip-address.d.ts +14 -0
- package/dist/recognizers/ip-address.d.ts.map +1 -0
- package/dist/recognizers/ip-address.js +183 -0
- package/dist/recognizers/ip-address.js.map +1 -0
- package/dist/recognizers/phone.d.ts +10 -0
- package/dist/recognizers/phone.d.ts.map +1 -0
- package/dist/recognizers/phone.js +145 -0
- package/dist/recognizers/phone.js.map +1 -0
- package/dist/recognizers/postal-code.d.ts +16 -0
- package/dist/recognizers/postal-code.d.ts.map +1 -0
- package/dist/recognizers/postal-code.js +42 -0
- package/dist/recognizers/postal-code.js.map +1 -0
- package/dist/recognizers/registry.d.ts +59 -0
- package/dist/recognizers/registry.d.ts.map +1 -0
- package/dist/recognizers/registry.js +113 -0
- package/dist/recognizers/registry.js.map +1 -0
- package/dist/recognizers/secrets/api-key.d.ts +6 -0
- package/dist/recognizers/secrets/api-key.d.ts.map +1 -0
- package/dist/recognizers/secrets/api-key.js +40 -0
- package/dist/recognizers/secrets/api-key.js.map +1 -0
- package/dist/recognizers/secrets/aws-credentials.d.ts +7 -0
- package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -0
- package/dist/recognizers/secrets/aws-credentials.js +68 -0
- package/dist/recognizers/secrets/aws-credentials.js.map +1 -0
- package/dist/recognizers/secrets/config-secret.d.ts +8 -0
- package/dist/recognizers/secrets/config-secret.d.ts.map +1 -0
- package/dist/recognizers/secrets/config-secret.js +59 -0
- package/dist/recognizers/secrets/config-secret.js.map +1 -0
- package/dist/recognizers/secrets/connection-string.d.ts +6 -0
- package/dist/recognizers/secrets/connection-string.d.ts.map +1 -0
- package/dist/recognizers/secrets/connection-string.js +40 -0
- package/dist/recognizers/secrets/connection-string.js.map +1 -0
- package/dist/recognizers/secrets/env-var.d.ts +8 -0
- package/dist/recognizers/secrets/env-var.d.ts.map +1 -0
- package/dist/recognizers/secrets/env-var.js +54 -0
- package/dist/recognizers/secrets/env-var.js.map +1 -0
- package/dist/recognizers/secrets/index.d.ts +22 -0
- package/dist/recognizers/secrets/index.d.ts.map +1 -0
- package/dist/recognizers/secrets/index.js +41 -0
- package/dist/recognizers/secrets/index.js.map +1 -0
- package/dist/recognizers/secrets/jwt.d.ts +6 -0
- package/dist/recognizers/secrets/jwt.d.ts.map +1 -0
- package/dist/recognizers/secrets/jwt.js +47 -0
- package/dist/recognizers/secrets/jwt.js.map +1 -0
- package/dist/recognizers/secrets/key-patterns.d.ts +10 -0
- package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -0
- package/dist/recognizers/secrets/key-patterns.js +15 -0
- package/dist/recognizers/secrets/key-patterns.js.map +1 -0
- package/dist/recognizers/secrets/literal-value.d.ts +10 -0
- package/dist/recognizers/secrets/literal-value.d.ts.map +1 -0
- package/dist/recognizers/secrets/literal-value.js +48 -0
- package/dist/recognizers/secrets/literal-value.js.map +1 -0
- package/dist/recognizers/secrets/private-key.d.ts +6 -0
- package/dist/recognizers/secrets/private-key.d.ts.map +1 -0
- package/dist/recognizers/secrets/private-key.js +16 -0
- package/dist/recognizers/secrets/private-key.js.map +1 -0
- package/dist/recognizers/url.d.ts +14 -0
- package/dist/recognizers/url.d.ts.map +1 -0
- package/dist/recognizers/url.js +121 -0
- package/dist/recognizers/url.js.map +1 -0
- package/dist/storage/browser.d.ts +13 -0
- package/dist/storage/browser.d.ts.map +1 -0
- package/dist/storage/browser.js +13 -0
- package/dist/storage/browser.js.map +1 -0
- package/dist/storage/in-memory.d.ts +60 -0
- package/dist/storage/in-memory.d.ts.map +1 -0
- package/dist/storage/in-memory.js +108 -0
- package/dist/storage/in-memory.js.map +1 -0
- package/dist/storage/index.d.ts +11 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +11 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +69 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +274 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/session-base.d.ts +34 -0
- package/dist/storage/session-base.d.ts.map +1 -0
- package/dist/storage/session-base.js +122 -0
- package/dist/storage/session-base.js.map +1 -0
- package/dist/storage/session.d.ts +19 -0
- package/dist/storage/session.d.ts.map +1 -0
- package/dist/storage/session.js +31 -0
- package/dist/storage/session.js.map +1 -0
- package/dist/storage/sqlite.d.ts +80 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +255 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/storage/types.d.ts +208 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +6 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/streaming/anonymizer-stream.d.ts +63 -0
- package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
- package/dist/streaming/anonymizer-stream.js +184 -0
- package/dist/streaming/anonymizer-stream.js.map +1 -0
- package/dist/streaming/index.d.ts +9 -0
- package/dist/streaming/index.d.ts.map +1 -0
- package/dist/streaming/index.js +8 -0
- package/dist/streaming/index.js.map +1 -0
- package/dist/streaming/sentence-buffer.d.ts +78 -0
- package/dist/streaming/sentence-buffer.d.ts.map +1 -0
- package/dist/streaming/sentence-buffer.js +239 -0
- package/dist/streaming/sentence-buffer.js.map +1 -0
- package/dist/streaming/stream-factory.d.ts +38 -0
- package/dist/streaming/stream-factory.d.ts.map +1 -0
- package/dist/streaming/stream-factory.js +69 -0
- package/dist/streaming/stream-factory.js.map +1 -0
- package/dist/streaming/types.d.ts +121 -0
- package/dist/streaming/types.d.ts.map +1 -0
- package/dist/streaming/types.js +5 -0
- package/dist/streaming/types.js.map +1 -0
- package/dist/types/index.d.ts +269 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +95 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/pii-types.d.ts +64 -0
- package/dist/types/pii-types.d.ts.map +1 -0
- package/dist/types/pii-types.js +153 -0
- package/dist/types/pii-types.js.map +1 -0
- package/dist/utils/iban-checksum.d.ts +23 -0
- package/dist/utils/iban-checksum.d.ts.map +1 -0
- package/dist/utils/iban-checksum.js +106 -0
- package/dist/utils/iban-checksum.js.map +1 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/json-walk.d.ts +21 -0
- package/dist/utils/json-walk.d.ts.map +1 -0
- package/dist/utils/json-walk.js +44 -0
- package/dist/utils/json-walk.js.map +1 -0
- package/dist/utils/luhn.d.ts +17 -0
- package/dist/utils/luhn.d.ts.map +1 -0
- package/dist/utils/luhn.js +55 -0
- package/dist/utils/luhn.js.map +1 -0
- package/dist/utils/offsets.d.ts +86 -0
- package/dist/utils/offsets.d.ts.map +1 -0
- package/dist/utils/offsets.js +124 -0
- package/dist/utils/offsets.js.map +1 -0
- package/dist/utils/path.d.ts +34 -0
- package/dist/utils/path.d.ts.map +1 -0
- package/dist/utils/path.js +96 -0
- package/dist/utils/path.js.map +1 -0
- package/dist/utils/regex.d.ts +11 -0
- package/dist/utils/regex.d.ts.map +1 -0
- package/dist/utils/regex.js +14 -0
- package/dist/utils/regex.js.map +1 -0
- package/dist/utils/storage-browser.d.ts +51 -0
- package/dist/utils/storage-browser.d.ts.map +1 -0
- package/dist/utils/storage-browser.js +381 -0
- package/dist/utils/storage-browser.js.map +1 -0
- package/dist/utils/storage-node.d.ts +51 -0
- package/dist/utils/storage-node.d.ts.map +1 -0
- package/dist/utils/storage-node.js +153 -0
- package/dist/utils/storage-node.js.map +1 -0
- package/dist/utils/storage.browser.d.ts +75 -0
- package/dist/utils/storage.browser.d.ts.map +1 -0
- package/dist/utils/storage.browser.js +69 -0
- package/dist/utils/storage.browser.js.map +1 -0
- package/dist/utils/storage.d.ts +73 -0
- package/dist/utils/storage.d.ts.map +1 -0
- package/dist/utils/storage.js +69 -0
- package/dist/utils/storage.js.map +1 -0
- package/index.js +14 -13
- package/package.json +19 -5
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NER Model Manager
|
|
3
|
+
* Handles automatic downloading and caching of NER models from Hugging Face Hub
|
|
4
|
+
* Browser-compatible using storage abstraction layer
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Available NER model variants
|
|
8
|
+
*/
|
|
9
|
+
export type NERModelMode = "standard" | "quantized" | "disabled" | "custom";
|
|
10
|
+
/**
|
|
11
|
+
* Model file info
|
|
12
|
+
*/
|
|
13
|
+
export interface ModelFileInfo {
|
|
14
|
+
/** Filename in the repo */
|
|
15
|
+
repoFile: string;
|
|
16
|
+
/** Local filename */
|
|
17
|
+
localFile: string;
|
|
18
|
+
/** Whether file is required */
|
|
19
|
+
required: boolean;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Model registry entry
|
|
23
|
+
*/
|
|
24
|
+
export interface ModelInfo {
|
|
25
|
+
/** Model identifier */
|
|
26
|
+
id: string;
|
|
27
|
+
/** Human-readable name */
|
|
28
|
+
name: string;
|
|
29
|
+
/** Description */
|
|
30
|
+
description: string;
|
|
31
|
+
/** Approximate size */
|
|
32
|
+
size: string;
|
|
33
|
+
/** Hugging Face repo ID */
|
|
34
|
+
hfRepo: string;
|
|
35
|
+
/** Subfolder in repo (for models with multiple variants) */
|
|
36
|
+
hfSubfolder?: string;
|
|
37
|
+
/** Files to download */
|
|
38
|
+
files: ModelFileInfo[];
|
|
39
|
+
/** Label map for this model */
|
|
40
|
+
labelMap: string[];
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Registry of available models hosted on Hugging Face Hub
|
|
44
|
+
*
|
|
45
|
+
* Using ELAN's ONNX exports which are optimized for JS/ONNX runtime
|
|
46
|
+
* https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx
|
|
47
|
+
*/
|
|
48
|
+
export declare const MODEL_REGISTRY: Record<"standard" | "quantized", ModelInfo>;
|
|
49
|
+
/**
|
|
50
|
+
* Gets the cache directory for models
|
|
51
|
+
* Uses platform-specific cache location (or virtual path in browser)
|
|
52
|
+
*/
|
|
53
|
+
export declare function getModelCacheDir(): Promise<string>;
|
|
54
|
+
/**
|
|
55
|
+
* Gets the path to a specific model variant
|
|
56
|
+
*/
|
|
57
|
+
export declare function getModelPath(mode: "standard" | "quantized"): Promise<string>;
|
|
58
|
+
/**
|
|
59
|
+
* Checks if a model is already downloaded
|
|
60
|
+
*/
|
|
61
|
+
export declare function isModelDownloaded(mode: "standard" | "quantized"): Promise<boolean>;
|
|
62
|
+
/**
|
|
63
|
+
* Progress callback for downloads
|
|
64
|
+
*/
|
|
65
|
+
export type DownloadProgressCallback = (progress: {
|
|
66
|
+
file: string;
|
|
67
|
+
bytesDownloaded: number;
|
|
68
|
+
totalBytes: number | null;
|
|
69
|
+
percent: number | null;
|
|
70
|
+
}) => void;
|
|
71
|
+
/**
|
|
72
|
+
* Downloads a model variant from Hugging Face Hub
|
|
73
|
+
*/
|
|
74
|
+
export declare function downloadModel(mode: "standard" | "quantized", onProgress?: DownloadProgressCallback, onStatus?: (status: string) => void): Promise<string>;
|
|
75
|
+
/**
|
|
76
|
+
* Gets model paths if available, or downloads if needed
|
|
77
|
+
*/
|
|
78
|
+
export declare function ensureModel(mode: "standard" | "quantized", options?: {
|
|
79
|
+
autoDownload?: boolean;
|
|
80
|
+
onProgress?: DownloadProgressCallback;
|
|
81
|
+
onStatus?: (status: string) => void;
|
|
82
|
+
}): Promise<{
|
|
83
|
+
modelPath: string;
|
|
84
|
+
vocabPath: string;
|
|
85
|
+
labelMapPath: string;
|
|
86
|
+
}>;
|
|
87
|
+
/**
|
|
88
|
+
* Loads the label map for a registry model, preferring the label_map.json
|
|
89
|
+
* persisted by ensureModel and falling back to the registry's map for the
|
|
90
|
+
* mode. The registry is authoritative — it is the source ensureModel writes
|
|
91
|
+
* to the file — so a missing or corrupted file never changes the decoded
|
|
92
|
+
* label order (a mismatched map silently misclassifies every NER entity).
|
|
93
|
+
*/
|
|
94
|
+
export declare function loadLabelMap(mode: "standard" | "quantized", labelMapPath: string): Promise<string[]>;
|
|
95
|
+
/**
|
|
96
|
+
* Clears cached models
|
|
97
|
+
*/
|
|
98
|
+
export declare function clearModelCache(mode?: "standard" | "quantized"): Promise<void>;
|
|
99
|
+
/**
|
|
100
|
+
* Lists downloaded models
|
|
101
|
+
*/
|
|
102
|
+
export declare function listDownloadedModels(): Promise<Array<{
|
|
103
|
+
mode: "standard" | "quantized";
|
|
104
|
+
path: string;
|
|
105
|
+
size: string;
|
|
106
|
+
}>>;
|
|
107
|
+
/**
|
|
108
|
+
* Gets info about available models
|
|
109
|
+
*/
|
|
110
|
+
export declare function getModelInfo(mode: "standard" | "quantized"): ModelInfo;
|
|
111
|
+
//# sourceMappingURL=model-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,UAAU,GAAG,WAAW,EAAE,SAAS,CAiDtE,CAAC;AAiCF;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAGxD;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,UAAU,GAAG,WAAW,GAC7B,OAAO,CAAC,OAAO,CAAC,CAyBlB;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,KAAK,IAAI,CAAC;AAsGX;;GAEG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,UAAU,CAAC,EAAE,wBAAwB,EACrC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC,CAsDjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,wBAAwB,CAAC;IACtC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC,GACL,OAAO,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAiCzE;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,UAAU,GAAG,WAAW,EAC9B,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAoBnB;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,GAC9B,OAAO,CAAC,IAAI,CAAC,CAUf;AAED;;GAEG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CACnD,KAAK,CAAC;IAAE,IAAI,EAAE,UAAU,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CACtE,CAgBA;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAEtE"}
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NER Model Manager
|
|
3
|
+
* Handles automatic downloading and caching of NER models from Hugging Face Hub
|
|
4
|
+
* Browser-compatible using storage abstraction layer
|
|
5
|
+
*/
|
|
6
|
+
import { getStorageProvider } from "#storage";
|
|
7
|
+
import { join, basename } from "../utils/path.js";
|
|
8
|
+
/**
|
|
9
|
+
* Registry of available models hosted on Hugging Face Hub
|
|
10
|
+
*
|
|
11
|
+
* Using ELAN's ONNX exports which are optimized for JS/ONNX runtime
|
|
12
|
+
* https://huggingface.co/tjruesch/xlm-roberta-base-ner-hrl-onnx
|
|
13
|
+
*/
|
|
14
|
+
export const MODEL_REGISTRY = {
|
|
15
|
+
standard: {
|
|
16
|
+
id: "xlm-roberta-ner-standard",
|
|
17
|
+
name: "XLM-RoBERTa NER (Standard)",
|
|
18
|
+
description: "Multilingual NER model supporting EN, DE, FR, ES, and more",
|
|
19
|
+
size: "~1.1 GB",
|
|
20
|
+
hfRepo: "tjruesch/xlm-roberta-base-ner-hrl-onnx",
|
|
21
|
+
hfSubfolder: "onnx",
|
|
22
|
+
files: [
|
|
23
|
+
{ repoFile: "model.onnx", localFile: "model.onnx", required: true },
|
|
24
|
+
],
|
|
25
|
+
labelMap: [
|
|
26
|
+
"O",
|
|
27
|
+
"B-DATE",
|
|
28
|
+
"I-DATE",
|
|
29
|
+
"B-PER",
|
|
30
|
+
"I-PER",
|
|
31
|
+
"B-ORG",
|
|
32
|
+
"I-ORG",
|
|
33
|
+
"B-LOC",
|
|
34
|
+
"I-LOC",
|
|
35
|
+
],
|
|
36
|
+
},
|
|
37
|
+
quantized: {
|
|
38
|
+
id: "xlm-roberta-ner-quantized",
|
|
39
|
+
name: "XLM-RoBERTa NER (Quantized)",
|
|
40
|
+
description: "Quantized version, ~4x smaller with minimal accuracy loss",
|
|
41
|
+
size: "~265 MB",
|
|
42
|
+
hfRepo: "tjruesch/xlm-roberta-base-ner-hrl-onnx",
|
|
43
|
+
hfSubfolder: "onnx",
|
|
44
|
+
files: [
|
|
45
|
+
{
|
|
46
|
+
repoFile: "model_quantized.onnx",
|
|
47
|
+
localFile: "model.onnx",
|
|
48
|
+
required: true,
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
labelMap: [
|
|
52
|
+
"O",
|
|
53
|
+
"B-DATE",
|
|
54
|
+
"I-DATE",
|
|
55
|
+
"B-PER",
|
|
56
|
+
"I-PER",
|
|
57
|
+
"B-ORG",
|
|
58
|
+
"I-ORG",
|
|
59
|
+
"B-LOC",
|
|
60
|
+
"I-LOC",
|
|
61
|
+
],
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Shared tokenizer files (same for both variants)
|
|
66
|
+
*/
|
|
67
|
+
const TOKENIZER_FILES = [
|
|
68
|
+
{ repoFile: "tokenizer.json", localFile: "tokenizer.json", required: true },
|
|
69
|
+
{
|
|
70
|
+
repoFile: "tokenizer_config.json",
|
|
71
|
+
localFile: "tokenizer_config.json",
|
|
72
|
+
required: false,
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
repoFile: "special_tokens_map.json",
|
|
76
|
+
localFile: "special_tokens_map.json",
|
|
77
|
+
required: false,
|
|
78
|
+
},
|
|
79
|
+
{ repoFile: "config.json", localFile: "config.json", required: false },
|
|
80
|
+
];
|
|
81
|
+
// Cached storage provider
|
|
82
|
+
let storageProvider = null;
|
|
83
|
+
/**
|
|
84
|
+
* Gets the storage provider (lazily initialized)
|
|
85
|
+
*/
|
|
86
|
+
async function getStorage() {
|
|
87
|
+
if (storageProvider === null) {
|
|
88
|
+
storageProvider = await getStorageProvider();
|
|
89
|
+
}
|
|
90
|
+
return storageProvider;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Gets the cache directory for models
|
|
94
|
+
* Uses platform-specific cache location (or virtual path in browser)
|
|
95
|
+
*/
|
|
96
|
+
export async function getModelCacheDir() {
|
|
97
|
+
const storage = await getStorage();
|
|
98
|
+
return storage.getCacheDir("models");
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Gets the path to a specific model variant
|
|
102
|
+
*/
|
|
103
|
+
export async function getModelPath(mode) {
|
|
104
|
+
const cacheDir = await getModelCacheDir();
|
|
105
|
+
return join(cacheDir, mode);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Checks if a model is already downloaded
|
|
109
|
+
*/
|
|
110
|
+
export async function isModelDownloaded(mode) {
|
|
111
|
+
const storage = await getStorage();
|
|
112
|
+
const modelDir = await getModelPath(mode);
|
|
113
|
+
const info = MODEL_REGISTRY[mode];
|
|
114
|
+
try {
|
|
115
|
+
// Check if model file exists
|
|
116
|
+
const modelFile = info.files.find((f) => f.required && f.localFile.includes("model"));
|
|
117
|
+
if (modelFile) {
|
|
118
|
+
const modelExists = await storage.exists(join(modelDir, modelFile.localFile));
|
|
119
|
+
if (!modelExists)
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
// Check if tokenizer exists
|
|
123
|
+
const tokenizerExists = await storage.exists(join(modelDir, "tokenizer.json"));
|
|
124
|
+
return tokenizerExists;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Builds a Hugging Face Hub download URL
|
|
132
|
+
*/
|
|
133
|
+
function getHuggingFaceUrl(repo, filename, subfolder) {
|
|
134
|
+
const filePath = subfolder !== undefined && subfolder !== ""
|
|
135
|
+
? `${subfolder}/${filename}`
|
|
136
|
+
: filename;
|
|
137
|
+
return `https://huggingface.co/${repo}/resolve/main/${filePath}`;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Downloads a file from URL and returns the data
|
|
141
|
+
*/
|
|
142
|
+
async function downloadFileData(url, fileName, onProgress) {
|
|
143
|
+
const response = await fetch(url, {
|
|
144
|
+
headers: {
|
|
145
|
+
"User-Agent": "rehydra/1.0.0",
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
if (!response.ok) {
|
|
149
|
+
if (response.status === 404) {
|
|
150
|
+
throw new Error(`File not found: ${url}`);
|
|
151
|
+
}
|
|
152
|
+
throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
|
|
153
|
+
}
|
|
154
|
+
const totalBytes = response.headers.get("content-length");
|
|
155
|
+
const total = totalBytes !== null && totalBytes !== "" ? parseInt(totalBytes, 10) : null;
|
|
156
|
+
const reader = response.body?.getReader();
|
|
157
|
+
if (reader === undefined) {
|
|
158
|
+
throw new Error("Response body is not readable");
|
|
159
|
+
}
|
|
160
|
+
const chunks = [];
|
|
161
|
+
let bytesDownloaded = 0;
|
|
162
|
+
// eslint-disable-next-line no-constant-condition
|
|
163
|
+
while (true) {
|
|
164
|
+
const result = await reader.read();
|
|
165
|
+
if (result.done)
|
|
166
|
+
break;
|
|
167
|
+
const value = result.value;
|
|
168
|
+
chunks.push(value);
|
|
169
|
+
bytesDownloaded += value.length;
|
|
170
|
+
if (onProgress) {
|
|
171
|
+
onProgress({
|
|
172
|
+
file: fileName,
|
|
173
|
+
bytesDownloaded,
|
|
174
|
+
totalBytes: total,
|
|
175
|
+
percent: total !== null && total > 0
|
|
176
|
+
? Math.round((bytesDownloaded / total) * 100)
|
|
177
|
+
: null,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
// Concatenate all chunks
|
|
182
|
+
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
|
|
183
|
+
const result = new Uint8Array(totalLength);
|
|
184
|
+
let offset = 0;
|
|
185
|
+
for (const chunk of chunks) {
|
|
186
|
+
result.set(chunk, offset);
|
|
187
|
+
offset += chunk.length;
|
|
188
|
+
}
|
|
189
|
+
return result;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Downloads a file from URL to storage
|
|
193
|
+
*/
|
|
194
|
+
async function downloadFile(url, destPath, onProgress) {
|
|
195
|
+
const storage = await getStorage();
|
|
196
|
+
const fileName = basename(destPath);
|
|
197
|
+
const data = await downloadFileData(url, fileName, onProgress);
|
|
198
|
+
await storage.writeFile(destPath, data);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Downloads a model variant from Hugging Face Hub
|
|
202
|
+
*/
|
|
203
|
+
export async function downloadModel(mode, onProgress, onStatus) {
|
|
204
|
+
const storage = await getStorage();
|
|
205
|
+
const info = MODEL_REGISTRY[mode];
|
|
206
|
+
const modelDir = await getModelPath(mode);
|
|
207
|
+
// Create directory
|
|
208
|
+
await storage.mkdir(modelDir);
|
|
209
|
+
onStatus?.(`Downloading ${info.name} from Hugging Face Hub...`);
|
|
210
|
+
onStatus?.(`Repository: ${info.hfRepo}`);
|
|
211
|
+
// Download model files
|
|
212
|
+
for (const file of info.files) {
|
|
213
|
+
const url = getHuggingFaceUrl(info.hfRepo, file.repoFile, info.hfSubfolder);
|
|
214
|
+
const destPath = join(modelDir, file.localFile);
|
|
215
|
+
onStatus?.(`Downloading ${file.repoFile}...`);
|
|
216
|
+
try {
|
|
217
|
+
await downloadFile(url, destPath, onProgress);
|
|
218
|
+
}
|
|
219
|
+
catch (e) {
|
|
220
|
+
if (file.required) {
|
|
221
|
+
throw new Error(`Failed to download required file ${file.repoFile}: ${String(e)}`);
|
|
222
|
+
}
|
|
223
|
+
// Optional files can fail silently
|
|
224
|
+
onStatus?.(`Skipping optional file ${file.repoFile}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
// Download tokenizer files (from repo root, not subfolder)
|
|
228
|
+
for (const file of TOKENIZER_FILES) {
|
|
229
|
+
const url = getHuggingFaceUrl(info.hfRepo, file.repoFile);
|
|
230
|
+
const destPath = join(modelDir, file.localFile);
|
|
231
|
+
try {
|
|
232
|
+
await downloadFile(url, destPath, onProgress);
|
|
233
|
+
}
|
|
234
|
+
catch (e) {
|
|
235
|
+
if (file.required) {
|
|
236
|
+
throw new Error(`Failed to download required file ${file.repoFile}: ${String(e)}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
// Write label map
|
|
241
|
+
const labelMapPath = join(modelDir, "label_map.json");
|
|
242
|
+
await storage.writeFile(labelMapPath, JSON.stringify(info.labelMap, null, 2));
|
|
243
|
+
onStatus?.("Download complete!");
|
|
244
|
+
return modelDir;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Gets model paths if available, or downloads if needed
|
|
248
|
+
*/
|
|
249
|
+
export async function ensureModel(mode, options = {}) {
|
|
250
|
+
const { autoDownload = true, onProgress, onStatus } = options;
|
|
251
|
+
const modelDir = await getModelPath(mode);
|
|
252
|
+
const info = MODEL_REGISTRY[mode];
|
|
253
|
+
// Check if already downloaded
|
|
254
|
+
const isDownloaded = await isModelDownloaded(mode);
|
|
255
|
+
if (!isDownloaded) {
|
|
256
|
+
if (!autoDownload) {
|
|
257
|
+
throw new Error(`NER model '${mode}' not found at ${modelDir}.\n\n` +
|
|
258
|
+
`To download automatically, use:\n` +
|
|
259
|
+
` createAnonymizer({ ner: { mode: '${mode}', autoDownload: true } })\n\n` +
|
|
260
|
+
`Or use regex-only mode:\n` +
|
|
261
|
+
` createAnonymizer({ ner: { mode: 'disabled' } })`);
|
|
262
|
+
}
|
|
263
|
+
await downloadModel(mode, onProgress, onStatus);
|
|
264
|
+
}
|
|
265
|
+
else {
|
|
266
|
+
onStatus?.(`Using cached model: ${info.name}`);
|
|
267
|
+
}
|
|
268
|
+
// Find model file
|
|
269
|
+
const modelFile = info.files.find((f) => f.localFile === "model.onnx");
|
|
270
|
+
return {
|
|
271
|
+
modelPath: join(modelDir, modelFile?.localFile ?? "model.onnx"),
|
|
272
|
+
vocabPath: join(modelDir, "tokenizer.json"),
|
|
273
|
+
labelMapPath: join(modelDir, "label_map.json"),
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Loads the label map for a registry model, preferring the label_map.json
|
|
278
|
+
* persisted by ensureModel and falling back to the registry's map for the
|
|
279
|
+
* mode. The registry is authoritative — it is the source ensureModel writes
|
|
280
|
+
* to the file — so a missing or corrupted file never changes the decoded
|
|
281
|
+
* label order (a mismatched map silently misclassifies every NER entity).
|
|
282
|
+
*/
|
|
283
|
+
export async function loadLabelMap(mode, labelMapPath) {
|
|
284
|
+
const registryLabelMap = MODEL_REGISTRY[mode].labelMap;
|
|
285
|
+
try {
|
|
286
|
+
const storage = await getStorage();
|
|
287
|
+
const content = await storage.readTextFile(labelMapPath);
|
|
288
|
+
const parsed = JSON.parse(content);
|
|
289
|
+
if (Array.isArray(parsed) &&
|
|
290
|
+
parsed.length > 0 &&
|
|
291
|
+
parsed.every((label) => typeof label === "string")) {
|
|
292
|
+
return parsed;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
catch {
|
|
296
|
+
// Fall through to registry map
|
|
297
|
+
}
|
|
298
|
+
return registryLabelMap;
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Clears cached models
|
|
302
|
+
*/
|
|
303
|
+
export async function clearModelCache(mode) {
|
|
304
|
+
const storage = await getStorage();
|
|
305
|
+
if (mode) {
|
|
306
|
+
const modelDir = await getModelPath(mode);
|
|
307
|
+
await storage.rm(modelDir, { recursive: true, force: true });
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
const cacheDir = await getModelCacheDir();
|
|
311
|
+
await storage.rm(cacheDir, { recursive: true, force: true });
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Lists downloaded models
|
|
316
|
+
*/
|
|
317
|
+
export async function listDownloadedModels() {
|
|
318
|
+
const models = [];
|
|
319
|
+
for (const mode of ["standard", "quantized"]) {
|
|
320
|
+
if (await isModelDownloaded(mode)) {
|
|
321
|
+
const modelPath = await getModelPath(mode);
|
|
322
|
+
const info = MODEL_REGISTRY[mode];
|
|
323
|
+
models.push({ mode, path: modelPath, size: info.size });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return models;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Gets info about available models
|
|
330
|
+
*/
|
|
331
|
+
export function getModelInfo(mode) {
|
|
332
|
+
return MODEL_REGISTRY[mode];
|
|
333
|
+
}
|
|
334
|
+
//# sourceMappingURL=model-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-manager.js","sourceRoot":"","sources":["../../src/ner/model-manager.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAwB,MAAM,UAAU,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAyClD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAgD;IACzE,QAAQ,EAAE;QACR,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,4BAA4B;QAClC,WAAW,EAAE,4DAA4D;QACzE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,wCAAwC;QAChD,WAAW,EAAE,MAAM;QACnB,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE;SACpE;QACD,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR;KACF;IACD,SAAS,EAAE;QACT,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,6BAA6B;QACnC,WAAW,EAAE,2DAA2D;QACxE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,wCAAwC;QAChD,WAAW,EAAE,MAAM;QACnB,KAAK,EAAE;YACL;gBACE,QAAQ,EAAE,sBAAsB;gBAChC,SAAS,EAAE,YAAY;gBACvB,QAAQ,EAAE,IAAI;aACf;SACF;QACD,QAAQ,EAAE;YACR,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR;KACF;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAoB;IACvC,EAAE,QAAQ,EAAE,gBAAgB,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC3E;QACE,QAAQ,EAAE,uBAAuB;QACjC,SAAS,EAAE,uBAAuB;QAClC,QAAQ,EAAE,KAAK;KAChB;IACD;QACE,QAAQ,EAAE,yBAAyB;QACnC,SAAS,EAAE,yBAAyB;QACpC,QAAQ,EAAE,KAAK;KAChB;IACD,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE;CACvE,CAAC;AAEF,0BAA0B;AAC1B,IAAI,eAAe,GAA2B,IAAI,CAAC;AAEnD;;GAEG;AACH,KAAK,UAAU,UAAU;IACvB,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,OAAO,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAA8B;IAE9B,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAC1C,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAA8B;IAE9B,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,CAAC;QACH,6BAA6B;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CACnD,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,MAAM,CACtC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,CACpC,CAAC;YACF,IAAI,CAAC,WAAW;gBAAE,OAAO,KAAK,CAAC;QACjC,CAAC;QAED,4BAA4B;QAC5B,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CACjC,CAAC;QACF,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAYD;;GAEG;AACH,SAAS,iBAAiB,CACxB,IAAY,EACZ,QAAgB,EAChB,SAAkB;IAElB,MAAM,QAAQ,GACZ,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE;QACzC,CAAC,CAAC,GAAG,SAAS,IAAI,QAAQ,EAAE;QAC5B,CAAC,CAAC,QAAQ,CAAC;IACf,OAAO,0BAA0B,IAAI,iBAAiB,QAAQ,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,QAAgB,EAChB,UAAqC;IAErC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,OAAO,EAAE;YACP,YAAY,EAAE,eAAe;SAC9B;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,sBAAsB,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC1D,MAAM,KAAK,GACT,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE7E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAC1C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,MAAM,CAAC,IAAI;YAAE,MAAM;QAEvB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAmB,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;QAEhC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC;gBACT,IAAI,EAAE,QAAQ;gBACd,eAAe;gBACf,UAAU,EAAE,KAAK;gBACjB,OAAO,EACL,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC;oBACzB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;oBAC7C,CAAC,CAAC,IAAI;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,GAAW,EACX,QAAgB,EAChB,UAAqC;IAErC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEpC,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/D,MAAM,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAA8B,EAC9B,UAAqC,EACrC,QAAmC;IAEnC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAE1C,mBAAmB;IACnB,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE9B,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,IAAI,2BAA2B,CAAC,CAAC;IAChE,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzC,uBAAuB;IACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhD,QAAQ,EAAE,CAAC,eAAe,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAClE,CAAC;YACJ,CAAC;YACD,mCAAmC;YACnC,QAAQ,EAAE,CAAC,0BAA0B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEhD,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAClE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE9E,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IAEjC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAA8B,EAC9B,UAII,EAAE;IAEN,MAAM,EAAE,YAAY,GAAG,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE9D,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,8BAA8B;IAC9B,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEnD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,kBAAkB,QAAQ,OAAO;gBACjD,mCAAmC;gBACnC,sCAAsC,IAAI,gCAAgC;gBAC1E,2BAA2B;gBAC3B,mDAAmD,CACtD,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,QAAQ,EAAE,CAAC,uBAAuB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC;IAEvE,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,IAAI,YAAY,CAAC;QAC/D,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC;QAC3C,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAA8B,EAC9B,YAAoB;IAEpB,MAAM,gBAAgB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;IAEvD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE5C,IACE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,MAAM,GAAG,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,EAClD,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAA+B;IAE/B,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IAEnC,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;QAC1C,MAAM,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IAGxC,MAAM,MAAM,GAIP,EAAE,CAAC;IAER,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,WAAW,CAAU,EAAE,CAAC;QACtD,IAAI,MAAM,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAA8B;IACzD,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NER Model Wrapper
|
|
3
|
+
* ONNX Runtime integration for Named Entity Recognition
|
|
4
|
+
* Supports both onnxruntime-node and onnxruntime-web
|
|
5
|
+
*/
|
|
6
|
+
import { type OrtSessionOptions } from "#onnx-runtime";
|
|
7
|
+
import { SpanMatch, AnonymizationPolicy } from "../types/index.js";
|
|
8
|
+
/**
|
|
9
|
+
* NER Model configuration
|
|
10
|
+
*/
|
|
11
|
+
export interface NERModelConfig {
|
|
12
|
+
/** Path to ONNX model file */
|
|
13
|
+
modelPath: string;
|
|
14
|
+
/** Path to vocabulary file */
|
|
15
|
+
vocabPath: string;
|
|
16
|
+
/** Label mapping (index -> label string) */
|
|
17
|
+
labelMap: string[];
|
|
18
|
+
/** Maximum sequence length */
|
|
19
|
+
maxLength: number;
|
|
20
|
+
/** Number of content tokens shared by adjacent long-input windows. */
|
|
21
|
+
windowOverlapTokens: number;
|
|
22
|
+
/**
|
|
23
|
+
* Maximum number of inference windows per input. Inputs that tokenize to
|
|
24
|
+
* more windows are scanned only up to this bound (onWindowCap fires once per
|
|
25
|
+
* capped inference pass — so twice for one input when caseFallback runs a
|
|
26
|
+
* second pass). Guards callers that feed very large opaque payloads
|
|
27
|
+
* (multi-megabyte base64 attachments) from unbounded inference cost.
|
|
28
|
+
* @default undefined (unbounded — full coverage)
|
|
29
|
+
*/
|
|
30
|
+
maxWindowsPerInput?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Called when an input exceeds maxWindowsPerInput and its tail is left
|
|
33
|
+
* unscanned. Callers should log/meter this — it is a coverage reduction,
|
|
34
|
+
* not an error. When absent, a console warning is emitted instead.
|
|
35
|
+
* `inputLength` is the full input character count; the true total window
|
|
36
|
+
* count is not reported because tokenization stops at the cap (computing
|
|
37
|
+
* it would require tokenizing the whole input, the cost the cap avoids).
|
|
38
|
+
*/
|
|
39
|
+
onWindowCap?: (info: {
|
|
40
|
+
scannedWindows: number;
|
|
41
|
+
inputLength: number;
|
|
42
|
+
}) => void;
|
|
43
|
+
/** Whether model expects lowercase input */
|
|
44
|
+
doLowerCase: boolean;
|
|
45
|
+
/** Model version for tracking */
|
|
46
|
+
modelVersion: string;
|
|
47
|
+
/** ONNX session options for performance tuning */
|
|
48
|
+
sessionOptions?: OrtSessionOptions;
|
|
49
|
+
/**
|
|
50
|
+
* Enable case-insensitive fallback pass for detecting lowercase names.
|
|
51
|
+
* Runs a second NER pass on title-cased text and merges new detections.
|
|
52
|
+
* @default false
|
|
53
|
+
*/
|
|
54
|
+
caseFallback?: boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Confidence penalty multiplier for case-fallback detections (0.0 - 1.0).
|
|
57
|
+
* Applied as: confidence * caseFallbackPenalty
|
|
58
|
+
* @default 0.85
|
|
59
|
+
*/
|
|
60
|
+
caseFallbackPenalty?: number;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* NER prediction result for a single text
|
|
64
|
+
*/
|
|
65
|
+
export interface NERPrediction {
|
|
66
|
+
/** Detected entity spans */
|
|
67
|
+
spans: SpanMatch[];
|
|
68
|
+
/** Processing time in ms */
|
|
69
|
+
processingTimeMs: number;
|
|
70
|
+
/** Model version used */
|
|
71
|
+
modelVersion: string;
|
|
72
|
+
/**
|
|
73
|
+
* True when the input exceeded maxWindowsPerInput and its tail was left
|
|
74
|
+
* unscanned by NER. Callers enforcing a fail-closed policy should treat
|
|
75
|
+
* this as incomplete coverage.
|
|
76
|
+
*/
|
|
77
|
+
truncated: boolean;
|
|
78
|
+
/**
|
|
79
|
+
* Number of leading input characters NER actually scanned — the end offset
|
|
80
|
+
* of the last content token. When not truncated this covers the whole input
|
|
81
|
+
* except any trailing whitespace/punctuation past the last token; when
|
|
82
|
+
* truncated, a caller can cut the input at this offset to keep only the
|
|
83
|
+
* fully-scanned portion.
|
|
84
|
+
*/
|
|
85
|
+
coverageChars: number;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Default label map for common NER models (CoNLL-style)
|
|
89
|
+
*/
|
|
90
|
+
export declare const DEFAULT_LABEL_MAP: string[];
|
|
91
|
+
/**
|
|
92
|
+
* NER Model wrapper for ONNX inference
|
|
93
|
+
*/
|
|
94
|
+
export declare class NERModel {
|
|
95
|
+
private ort;
|
|
96
|
+
private session;
|
|
97
|
+
private tokenizer;
|
|
98
|
+
private config;
|
|
99
|
+
private isLoaded;
|
|
100
|
+
constructor(config: NERModelConfig);
|
|
101
|
+
/**
|
|
102
|
+
* Loads the model and tokenizer
|
|
103
|
+
*/
|
|
104
|
+
load(): Promise<void>;
|
|
105
|
+
/**
|
|
106
|
+
* Runs a single NER pass: tokenize, infer, decode BIO tags.
|
|
107
|
+
* @param inputText - Text to tokenize and feed to the model
|
|
108
|
+
* @param originalText - Original text used for extracting entity text (may differ in casing)
|
|
109
|
+
*/
|
|
110
|
+
private runNERPass;
|
|
111
|
+
/**
|
|
112
|
+
* Predicts entities in text
|
|
113
|
+
*/
|
|
114
|
+
predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
|
|
115
|
+
/**
|
|
116
|
+
* Runs ONNX inference
|
|
117
|
+
*/
|
|
118
|
+
private runInference;
|
|
119
|
+
/**
|
|
120
|
+
* Processes model logits to extract labels and confidences
|
|
121
|
+
*/
|
|
122
|
+
private processLogits;
|
|
123
|
+
/**
|
|
124
|
+
* Gets minimum confidence threshold from policy
|
|
125
|
+
*/
|
|
126
|
+
private getMinConfidence;
|
|
127
|
+
/**
|
|
128
|
+
* Gets model version
|
|
129
|
+
*/
|
|
130
|
+
get version(): string;
|
|
131
|
+
/**
|
|
132
|
+
* Checks if model is loaded
|
|
133
|
+
*/
|
|
134
|
+
get loaded(): boolean;
|
|
135
|
+
/**
|
|
136
|
+
* Disposes of model resources
|
|
137
|
+
*/
|
|
138
|
+
dispose(): Promise<void>;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Creates a NER model instance with configuration
|
|
142
|
+
*/
|
|
143
|
+
export declare function createNERModel(config: Partial<NERModelConfig> & {
|
|
144
|
+
modelPath: string;
|
|
145
|
+
vocabPath: string;
|
|
146
|
+
}): NERModel;
|
|
147
|
+
/**
|
|
148
|
+
* NER Model stub for when no model is available
|
|
149
|
+
* Returns empty results - useful for regex-only mode
|
|
150
|
+
*/
|
|
151
|
+
export declare class NERModelStub {
|
|
152
|
+
readonly version = "stub-1.0.0";
|
|
153
|
+
readonly loaded = true;
|
|
154
|
+
load(): Promise<void>;
|
|
155
|
+
predict(_text: string, _policy?: AnonymizationPolicy): Promise<NERPrediction>;
|
|
156
|
+
dispose(): Promise<void>;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Creates a stub NER model (for testing or regex-only mode)
|
|
160
|
+
*/
|
|
161
|
+
export declare function createNERModelStub(): NERModelStub;
|
|
162
|
+
/**
|
|
163
|
+
* NER model interface for dependency injection
|
|
164
|
+
*/
|
|
165
|
+
export interface INERModel {
|
|
166
|
+
readonly version: string;
|
|
167
|
+
readonly loaded: boolean;
|
|
168
|
+
load(): Promise<void>;
|
|
169
|
+
predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
|
|
170
|
+
dispose(): Promise<void>;
|
|
171
|
+
}
|
|
172
|
+
//# sourceMappingURL=ner-model.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ner-model.d.ts","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAoBnE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAC9E,4CAA4C;IAC5C,WAAW,EAAE,OAAO,CAAC;IACrB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,cAAc,CAAC,EAAE,iBAAiB,CAAC;IACnC;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,4BAA4B;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;IACnB;;;;;;OAMG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,UAU7B,CAAC;AA8BF;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,GAAG,CAA2B;IACtC,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C3B;;;;OAIG;YACW,UAAU;IAkExB;;OAEG;IACG,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC;IAmEzB;;OAEG;YACW,YAAY;IAiE1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqCrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAexB;;OAEG;IACH,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAOzB;AAsKD;;GAEG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzE,QAAQ,CA0BV;AAED;;;GAGG;AACH,qBAAa,YAAY;IACvB,QAAQ,CAAC,OAAO,gBAAgB;IAChC,QAAQ,CAAC,MAAM,QAAQ;IAEjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,OAAO,CACL,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,aAAa,CAAC;IAUzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAIzB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,YAAY,CAEjD;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5E,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}
|