opencode-rehydar 0.14.2 → 0.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +54 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +74 -0
- package/dist/browser.js.map +1 -0
- package/dist/core/anonymizer.d.ts +309 -0
- package/dist/core/anonymizer.d.ts.map +1 -0
- package/dist/core/anonymizer.js +555 -0
- package/dist/core/anonymizer.js.map +1 -0
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +6 -0
- package/dist/core/index.js.map +1 -0
- package/dist/crypto/index.d.ts +6 -0
- package/dist/crypto/index.d.ts.map +1 -0
- package/dist/crypto/index.js +6 -0
- package/dist/crypto/index.js.map +1 -0
- package/dist/crypto/pii-map-crypto.d.ts +114 -0
- package/dist/crypto/pii-map-crypto.d.ts.map +1 -0
- package/dist/crypto/pii-map-crypto.js +228 -0
- package/dist/crypto/pii-map-crypto.js.map +1 -0
- package/dist/index.d.ts +53 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +72 -0
- package/dist/index.js.map +1 -0
- package/dist/ner/bio-decoder.d.ts +64 -0
- package/dist/ner/bio-decoder.d.ts.map +1 -0
- package/dist/ner/bio-decoder.js +218 -0
- package/dist/ner/bio-decoder.js.map +1 -0
- package/dist/ner/index.d.ts +12 -0
- package/dist/ner/index.d.ts.map +1 -0
- package/dist/ner/index.js +12 -0
- package/dist/ner/index.js.map +1 -0
- package/dist/ner/inference-server-client.d.ts +81 -0
- package/dist/ner/inference-server-client.d.ts.map +1 -0
- package/dist/ner/inference-server-client.js +120 -0
- package/dist/ner/inference-server-client.js.map +1 -0
- package/dist/ner/inference-server-ner-model.d.ts +43 -0
- package/dist/ner/inference-server-ner-model.d.ts.map +1 -0
- package/dist/ner/inference-server-ner-model.js +119 -0
- package/dist/ner/inference-server-ner-model.js.map +1 -0
- package/dist/ner/model-manager.d.ts +111 -0
- package/dist/ner/model-manager.d.ts.map +1 -0
- package/dist/ner/model-manager.js +334 -0
- package/dist/ner/model-manager.js.map +1 -0
- package/dist/ner/ner-model.d.ts +172 -0
- package/dist/ner/ner-model.d.ts.map +1 -0
- package/dist/ner/ner-model.js +514 -0
- package/dist/ner/ner-model.js.map +1 -0
- package/dist/ner/onnx-runtime.browser.d.ts +46 -0
- package/dist/ner/onnx-runtime.browser.d.ts.map +1 -0
- package/dist/ner/onnx-runtime.browser.js +90 -0
- package/dist/ner/onnx-runtime.browser.js.map +1 -0
- package/dist/ner/onnx-runtime.d.ts +72 -0
- package/dist/ner/onnx-runtime.d.ts.map +1 -0
- package/dist/ner/onnx-runtime.js +132 -0
- package/dist/ner/onnx-runtime.js.map +1 -0
- package/dist/ner/token-window.d.ts +19 -0
- package/dist/ner/token-window.d.ts.map +1 -0
- package/dist/ner/token-window.js +101 -0
- package/dist/ner/token-window.js.map +1 -0
- package/dist/ner/tokenizer.d.ts +142 -0
- package/dist/ner/tokenizer.d.ts.map +1 -0
- package/dist/ner/tokenizer.js +362 -0
- package/dist/ner/tokenizer.js.map +1 -0
- package/dist/opencode-plugin/git-identity.d.ts +3 -0
- package/dist/opencode-plugin/git-identity.d.ts.map +1 -0
- package/dist/opencode-plugin/git-identity.js +37 -0
- package/dist/opencode-plugin/git-identity.js.map +1 -0
- package/dist/opencode-plugin/github-username.d.ts +3 -0
- package/dist/opencode-plugin/github-username.d.ts.map +1 -0
- package/dist/opencode-plugin/github-username.js +41 -0
- package/dist/opencode-plugin/github-username.js.map +1 -0
- package/dist/opencode-plugin/identity-recognizer.d.ts +5 -0
- package/dist/opencode-plugin/identity-recognizer.d.ts.map +1 -0
- package/dist/opencode-plugin/identity-recognizer.js +21 -0
- package/dist/opencode-plugin/identity-recognizer.js.map +1 -0
- package/dist/opencode-plugin/index.d.ts +7 -0
- package/dist/opencode-plugin/index.d.ts.map +1 -0
- package/dist/opencode-plugin/index.js +6 -0
- package/dist/opencode-plugin/index.js.map +1 -0
- package/dist/opencode-plugin/plugin.d.ts +39 -0
- package/dist/opencode-plugin/plugin.d.ts.map +1 -0
- package/dist/opencode-plugin/plugin.js +294 -0
- package/dist/opencode-plugin/plugin.js.map +1 -0
- package/dist/opencode-plugin/types.d.ts +33 -0
- package/dist/opencode-plugin/types.d.ts.map +1 -0
- package/dist/opencode-plugin/types.js +5 -0
- package/dist/opencode-plugin/types.js.map +1 -0
- package/dist/opencode-plugin/v2/anon-state.d.ts +12 -0
- package/dist/opencode-plugin/v2/anon-state.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/anon-state.js +14 -0
- package/dist/opencode-plugin/v2/anon-state.js.map +1 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.d.ts +13 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.js +33 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.js.map +1 -0
- package/dist/opencode-plugin/v2/index.d.ts +17 -0
- package/dist/opencode-plugin/v2/index.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/index.js +17 -0
- package/dist/opencode-plugin/v2/index.js.map +1 -0
- package/dist/opencode-plugin/v2/logger.d.ts +24 -0
- package/dist/opencode-plugin/v2/logger.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/logger.js +73 -0
- package/dist/opencode-plugin/v2/logger.js.map +1 -0
- package/dist/opencode-plugin/v2/options.d.ts +63 -0
- package/dist/opencode-plugin/v2/options.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/options.js +183 -0
- package/dist/opencode-plugin/v2/options.js.map +1 -0
- package/dist/opencode-plugin/v2/register-hooks.d.ts +43 -0
- package/dist/opencode-plugin/v2/register-hooks.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/register-hooks.js +160 -0
- package/dist/opencode-plugin/v2/register-hooks.js.map +1 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.d.ts +101 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.js +147 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.js.map +1 -0
- package/dist/opencode-plugin/v2/response-rehydrate.d.ts +78 -0
- package/dist/opencode-plugin/v2/response-rehydrate.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/response-rehydrate.js +210 -0
- package/dist/opencode-plugin/v2/response-rehydrate.js.map +1 -0
- package/dist/opencode-plugin/v2/scrub-messages.d.ts +39 -0
- package/dist/opencode-plugin/v2/scrub-messages.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/scrub-messages.js +260 -0
- package/dist/opencode-plugin/v2/scrub-messages.js.map +1 -0
- package/dist/opencode-plugin/v2/sessions.d.ts +17 -0
- package/dist/opencode-plugin/v2/sessions.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/sessions.js +23 -0
- package/dist/opencode-plugin/v2/sessions.js.map +1 -0
- package/dist/opencode-plugin/v2/sse.d.ts +101 -0
- package/dist/opencode-plugin/v2/sse.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/sse.js +235 -0
- package/dist/opencode-plugin/v2/sse.js.map +1 -0
- package/dist/opencode-plugin/v2/types.d.ts +188 -0
- package/dist/opencode-plugin/v2/types.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/types.js +10 -0
- package/dist/opencode-plugin/v2/types.js.map +1 -0
- package/dist/opencode-plugin/vcs-command.d.ts +3 -0
- package/dist/opencode-plugin/vcs-command.d.ts.map +1 -0
- package/dist/opencode-plugin/vcs-command.js +34 -0
- package/dist/opencode-plugin/vcs-command.js.map +1 -0
- package/dist/pipeline/index.d.ts +12 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/pipeline/index.js +12 -0
- package/dist/pipeline/index.js.map +1 -0
- package/dist/pipeline/prenormalize.d.ts +48 -0
- package/dist/pipeline/prenormalize.d.ts.map +1 -0
- package/dist/pipeline/prenormalize.js +94 -0
- package/dist/pipeline/prenormalize.js.map +1 -0
- package/dist/pipeline/resolver.d.ts +56 -0
- package/dist/pipeline/resolver.d.ts.map +1 -0
- package/dist/pipeline/resolver.js +239 -0
- package/dist/pipeline/resolver.js.map +1 -0
- package/dist/pipeline/semantic-data-loader.d.ts +171 -0
- package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
- package/dist/pipeline/semantic-data-loader.js +677 -0
- package/dist/pipeline/semantic-data-loader.js.map +1 -0
- package/dist/pipeline/semantic-enricher.d.ts +112 -0
- package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
- package/dist/pipeline/semantic-enricher.js +322 -0
- package/dist/pipeline/semantic-enricher.js.map +1 -0
- package/dist/pipeline/tagger.d.ts +121 -0
- package/dist/pipeline/tagger.d.ts.map +1 -0
- package/dist/pipeline/tagger.js +545 -0
- package/dist/pipeline/tagger.js.map +1 -0
- package/dist/pipeline/title-extractor.d.ts +79 -0
- package/dist/pipeline/title-extractor.d.ts.map +1 -0
- package/dist/pipeline/title-extractor.js +801 -0
- package/dist/pipeline/title-extractor.js.map +1 -0
- package/dist/pipeline/validator.d.ts +65 -0
- package/dist/pipeline/validator.d.ts.map +1 -0
- package/dist/pipeline/validator.js +292 -0
- package/dist/pipeline/validator.js.map +1 -0
- package/dist/proxy/index.d.ts +13 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +12 -0
- package/dist/proxy/index.js.map +1 -0
- package/dist/proxy/providers/anthropic.d.ts +28 -0
- package/dist/proxy/providers/anthropic.d.ts.map +1 -0
- package/dist/proxy/providers/anthropic.js +265 -0
- package/dist/proxy/providers/anthropic.js.map +1 -0
- package/dist/proxy/providers/index.d.ts +19 -0
- package/dist/proxy/providers/index.d.ts.map +1 -0
- package/dist/proxy/providers/index.js +41 -0
- package/dist/proxy/providers/index.js.map +1 -0
- package/dist/proxy/providers/openai.d.ts +27 -0
- package/dist/proxy/providers/openai.d.ts.map +1 -0
- package/dist/proxy/providers/openai.js +203 -0
- package/dist/proxy/providers/openai.js.map +1 -0
- package/dist/proxy/providers/streaming-tool-loop.d.ts +12 -0
- package/dist/proxy/providers/streaming-tool-loop.d.ts.map +1 -0
- package/dist/proxy/providers/streaming-tool-loop.js +223 -0
- package/dist/proxy/providers/streaming-tool-loop.js.map +1 -0
- package/dist/proxy/providers/types.d.ts +109 -0
- package/dist/proxy/providers/types.d.ts.map +1 -0
- package/dist/proxy/providers/types.js +2 -0
- package/dist/proxy/providers/types.js.map +1 -0
- package/dist/proxy/proxy-server.d.ts +61 -0
- package/dist/proxy/proxy-server.d.ts.map +1 -0
- package/dist/proxy/proxy-server.js +146 -0
- package/dist/proxy/proxy-server.js.map +1 -0
- package/dist/proxy/rehydra-fetch.d.ts +35 -0
- package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
- package/dist/proxy/rehydra-fetch.js +631 -0
- package/dist/proxy/rehydra-fetch.js.map +1 -0
- package/dist/proxy/rehydra-proxy.d.ts +40 -0
- package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
- package/dist/proxy/rehydra-proxy.js +150 -0
- package/dist/proxy/rehydra-proxy.js.map +1 -0
- package/dist/proxy/sse-parser.d.ts +59 -0
- package/dist/proxy/sse-parser.d.ts.map +1 -0
- package/dist/proxy/sse-parser.js +112 -0
- package/dist/proxy/sse-parser.js.map +1 -0
- package/dist/proxy/system-instruction.d.ts +19 -0
- package/dist/proxy/system-instruction.d.ts.map +1 -0
- package/dist/proxy/system-instruction.js +33 -0
- package/dist/proxy/system-instruction.js.map +1 -0
- package/dist/proxy/tool-loop-stream.d.ts +7 -0
- package/dist/proxy/tool-loop-stream.d.ts.map +1 -0
- package/dist/proxy/tool-loop-stream.js +65 -0
- package/dist/proxy/tool-loop-stream.js.map +1 -0
- package/dist/proxy/types.d.ts +138 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +5 -0
- package/dist/proxy/types.js.map +1 -0
- package/dist/proxy/wrap-client.d.ts +47 -0
- package/dist/proxy/wrap-client.d.ts.map +1 -0
- package/dist/proxy/wrap-client.js +70 -0
- package/dist/proxy/wrap-client.js.map +1 -0
- package/dist/recognizers/address.d.ts +2 -0
- package/dist/recognizers/address.d.ts.map +1 -0
- package/dist/recognizers/address.js +23 -0
- package/dist/recognizers/address.js.map +1 -0
- package/dist/recognizers/base.d.ts +78 -0
- package/dist/recognizers/base.d.ts.map +1 -0
- package/dist/recognizers/base.js +100 -0
- package/dist/recognizers/base.js.map +1 -0
- package/dist/recognizers/bic-swift.d.ts +10 -0
- package/dist/recognizers/bic-swift.d.ts.map +1 -0
- package/dist/recognizers/bic-swift.js +107 -0
- package/dist/recognizers/bic-swift.js.map +1 -0
- package/dist/recognizers/credit-card.d.ts +32 -0
- package/dist/recognizers/credit-card.d.ts.map +1 -0
- package/dist/recognizers/credit-card.js +160 -0
- package/dist/recognizers/credit-card.js.map +1 -0
- package/dist/recognizers/custom-id.d.ts +28 -0
- package/dist/recognizers/custom-id.d.ts.map +1 -0
- package/dist/recognizers/custom-id.js +116 -0
- package/dist/recognizers/custom-id.js.map +1 -0
- package/dist/recognizers/date.d.ts +10 -0
- package/dist/recognizers/date.d.ts.map +1 -0
- package/dist/recognizers/date.js +228 -0
- package/dist/recognizers/date.js.map +1 -0
- package/dist/recognizers/email.d.ts +10 -0
- package/dist/recognizers/email.d.ts.map +1 -0
- package/dist/recognizers/email.js +75 -0
- package/dist/recognizers/email.js.map +1 -0
- package/dist/recognizers/iban.d.ts +14 -0
- package/dist/recognizers/iban.d.ts.map +1 -0
- package/dist/recognizers/iban.js +67 -0
- package/dist/recognizers/iban.js.map +1 -0
- package/dist/recognizers/index.d.ts +25 -0
- package/dist/recognizers/index.d.ts.map +1 -0
- package/dist/recognizers/index.js +53 -0
- package/dist/recognizers/index.js.map +1 -0
- package/dist/recognizers/ip-address.d.ts +14 -0
- package/dist/recognizers/ip-address.d.ts.map +1 -0
- package/dist/recognizers/ip-address.js +183 -0
- package/dist/recognizers/ip-address.js.map +1 -0
- package/dist/recognizers/phone.d.ts +10 -0
- package/dist/recognizers/phone.d.ts.map +1 -0
- package/dist/recognizers/phone.js +145 -0
- package/dist/recognizers/phone.js.map +1 -0
- package/dist/recognizers/postal-code.d.ts +16 -0
- package/dist/recognizers/postal-code.d.ts.map +1 -0
- package/dist/recognizers/postal-code.js +42 -0
- package/dist/recognizers/postal-code.js.map +1 -0
- package/dist/recognizers/registry.d.ts +59 -0
- package/dist/recognizers/registry.d.ts.map +1 -0
- package/dist/recognizers/registry.js +113 -0
- package/dist/recognizers/registry.js.map +1 -0
- package/dist/recognizers/secrets/api-key.d.ts +6 -0
- package/dist/recognizers/secrets/api-key.d.ts.map +1 -0
- package/dist/recognizers/secrets/api-key.js +40 -0
- package/dist/recognizers/secrets/api-key.js.map +1 -0
- package/dist/recognizers/secrets/aws-credentials.d.ts +7 -0
- package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -0
- package/dist/recognizers/secrets/aws-credentials.js +68 -0
- package/dist/recognizers/secrets/aws-credentials.js.map +1 -0
- package/dist/recognizers/secrets/config-secret.d.ts +8 -0
- package/dist/recognizers/secrets/config-secret.d.ts.map +1 -0
- package/dist/recognizers/secrets/config-secret.js +59 -0
- package/dist/recognizers/secrets/config-secret.js.map +1 -0
- package/dist/recognizers/secrets/connection-string.d.ts +6 -0
- package/dist/recognizers/secrets/connection-string.d.ts.map +1 -0
- package/dist/recognizers/secrets/connection-string.js +40 -0
- package/dist/recognizers/secrets/connection-string.js.map +1 -0
- package/dist/recognizers/secrets/env-var.d.ts +8 -0
- package/dist/recognizers/secrets/env-var.d.ts.map +1 -0
- package/dist/recognizers/secrets/env-var.js +54 -0
- package/dist/recognizers/secrets/env-var.js.map +1 -0
- package/dist/recognizers/secrets/index.d.ts +22 -0
- package/dist/recognizers/secrets/index.d.ts.map +1 -0
- package/dist/recognizers/secrets/index.js +41 -0
- package/dist/recognizers/secrets/index.js.map +1 -0
- package/dist/recognizers/secrets/jwt.d.ts +6 -0
- package/dist/recognizers/secrets/jwt.d.ts.map +1 -0
- package/dist/recognizers/secrets/jwt.js +47 -0
- package/dist/recognizers/secrets/jwt.js.map +1 -0
- package/dist/recognizers/secrets/key-patterns.d.ts +10 -0
- package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -0
- package/dist/recognizers/secrets/key-patterns.js +15 -0
- package/dist/recognizers/secrets/key-patterns.js.map +1 -0
- package/dist/recognizers/secrets/literal-value.d.ts +10 -0
- package/dist/recognizers/secrets/literal-value.d.ts.map +1 -0
- package/dist/recognizers/secrets/literal-value.js +48 -0
- package/dist/recognizers/secrets/literal-value.js.map +1 -0
- package/dist/recognizers/secrets/private-key.d.ts +6 -0
- package/dist/recognizers/secrets/private-key.d.ts.map +1 -0
- package/dist/recognizers/secrets/private-key.js +16 -0
- package/dist/recognizers/secrets/private-key.js.map +1 -0
- package/dist/recognizers/url.d.ts +14 -0
- package/dist/recognizers/url.d.ts.map +1 -0
- package/dist/recognizers/url.js +121 -0
- package/dist/recognizers/url.js.map +1 -0
- package/dist/storage/browser.d.ts +13 -0
- package/dist/storage/browser.d.ts.map +1 -0
- package/dist/storage/browser.js +13 -0
- package/dist/storage/browser.js.map +1 -0
- package/dist/storage/in-memory.d.ts +60 -0
- package/dist/storage/in-memory.d.ts.map +1 -0
- package/dist/storage/in-memory.js +108 -0
- package/dist/storage/in-memory.js.map +1 -0
- package/dist/storage/index.d.ts +11 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +11 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +69 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +274 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/session-base.d.ts +34 -0
- package/dist/storage/session-base.d.ts.map +1 -0
- package/dist/storage/session-base.js +122 -0
- package/dist/storage/session-base.js.map +1 -0
- package/dist/storage/session.d.ts +19 -0
- package/dist/storage/session.d.ts.map +1 -0
- package/dist/storage/session.js +31 -0
- package/dist/storage/session.js.map +1 -0
- package/dist/storage/sqlite.d.ts +80 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +255 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/storage/types.d.ts +208 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +6 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/streaming/anonymizer-stream.d.ts +63 -0
- package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
- package/dist/streaming/anonymizer-stream.js +184 -0
- package/dist/streaming/anonymizer-stream.js.map +1 -0
- package/dist/streaming/index.d.ts +9 -0
- package/dist/streaming/index.d.ts.map +1 -0
- package/dist/streaming/index.js +8 -0
- package/dist/streaming/index.js.map +1 -0
- package/dist/streaming/sentence-buffer.d.ts +78 -0
- package/dist/streaming/sentence-buffer.d.ts.map +1 -0
- package/dist/streaming/sentence-buffer.js +239 -0
- package/dist/streaming/sentence-buffer.js.map +1 -0
- package/dist/streaming/stream-factory.d.ts +38 -0
- package/dist/streaming/stream-factory.d.ts.map +1 -0
- package/dist/streaming/stream-factory.js +69 -0
- package/dist/streaming/stream-factory.js.map +1 -0
- package/dist/streaming/types.d.ts +121 -0
- package/dist/streaming/types.d.ts.map +1 -0
- package/dist/streaming/types.js +5 -0
- package/dist/streaming/types.js.map +1 -0
- package/dist/types/index.d.ts +269 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +95 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/pii-types.d.ts +64 -0
- package/dist/types/pii-types.d.ts.map +1 -0
- package/dist/types/pii-types.js +153 -0
- package/dist/types/pii-types.js.map +1 -0
- package/dist/utils/iban-checksum.d.ts +23 -0
- package/dist/utils/iban-checksum.d.ts.map +1 -0
- package/dist/utils/iban-checksum.js +106 -0
- package/dist/utils/iban-checksum.js.map +1 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/json-walk.d.ts +21 -0
- package/dist/utils/json-walk.d.ts.map +1 -0
- package/dist/utils/json-walk.js +44 -0
- package/dist/utils/json-walk.js.map +1 -0
- package/dist/utils/luhn.d.ts +17 -0
- package/dist/utils/luhn.d.ts.map +1 -0
- package/dist/utils/luhn.js +55 -0
- package/dist/utils/luhn.js.map +1 -0
- package/dist/utils/offsets.d.ts +86 -0
- package/dist/utils/offsets.d.ts.map +1 -0
- package/dist/utils/offsets.js +124 -0
- package/dist/utils/offsets.js.map +1 -0
- package/dist/utils/path.d.ts +34 -0
- package/dist/utils/path.d.ts.map +1 -0
- package/dist/utils/path.js +96 -0
- package/dist/utils/path.js.map +1 -0
- package/dist/utils/regex.d.ts +11 -0
- package/dist/utils/regex.d.ts.map +1 -0
- package/dist/utils/regex.js +14 -0
- package/dist/utils/regex.js.map +1 -0
- package/dist/utils/storage-browser.d.ts +51 -0
- package/dist/utils/storage-browser.d.ts.map +1 -0
- package/dist/utils/storage-browser.js +381 -0
- package/dist/utils/storage-browser.js.map +1 -0
- package/dist/utils/storage-node.d.ts +51 -0
- package/dist/utils/storage-node.d.ts.map +1 -0
- package/dist/utils/storage-node.js +153 -0
- package/dist/utils/storage-node.js.map +1 -0
- package/dist/utils/storage.browser.d.ts +75 -0
- package/dist/utils/storage.browser.d.ts.map +1 -0
- package/dist/utils/storage.browser.js +69 -0
- package/dist/utils/storage.browser.js.map +1 -0
- package/dist/utils/storage.d.ts +73 -0
- package/dist/utils/storage.d.ts.map +1 -0
- package/dist/utils/storage.js +69 -0
- package/dist/utils/storage.js.map +1 -0
- package/index.js +6 -4
- package/package.json +19 -5
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BIO Tag Decoder
|
|
3
|
+
* Converts BIO-tagged token sequences to entity spans
|
|
4
|
+
*/
|
|
5
|
+
import { PIIType, DetectionSource } from '../types/index.js';
|
|
6
|
+
import { getPIITypeFromNERLabel } from '../types/pii-types.js';
|
|
7
|
+
/**
|
|
8
|
+
* BIO tag types
|
|
9
|
+
*/
|
|
10
|
+
export var BIOTag;
|
|
11
|
+
(function (BIOTag) {
|
|
12
|
+
/** Beginning of an entity */
|
|
13
|
+
BIOTag["B"] = "B";
|
|
14
|
+
/** Inside an entity (continuation) */
|
|
15
|
+
BIOTag["I"] = "I";
|
|
16
|
+
/** Outside any entity */
|
|
17
|
+
BIOTag["O"] = "O";
|
|
18
|
+
})(BIOTag || (BIOTag = {}));
|
|
19
|
+
/**
|
|
20
|
+
* Parses a BIO label string (e.g., "B-PER", "I-ORG", "O")
|
|
21
|
+
*/
|
|
22
|
+
export function parseBIOLabel(label) {
|
|
23
|
+
if (label === 'O' || label === '[PAD]' || label === '[CLS]' || label === '[SEP]') {
|
|
24
|
+
return { tag: BIOTag.O, entityType: null };
|
|
25
|
+
}
|
|
26
|
+
const parts = label.split('-');
|
|
27
|
+
if (parts.length !== 2) {
|
|
28
|
+
return { tag: BIOTag.O, entityType: null };
|
|
29
|
+
}
|
|
30
|
+
const [tagStr, entityType] = parts;
|
|
31
|
+
let tag;
|
|
32
|
+
switch (tagStr?.toUpperCase()) {
|
|
33
|
+
case 'B':
|
|
34
|
+
tag = BIOTag.B;
|
|
35
|
+
break;
|
|
36
|
+
case 'I':
|
|
37
|
+
tag = BIOTag.I;
|
|
38
|
+
break;
|
|
39
|
+
default:
|
|
40
|
+
return { tag: BIOTag.O, entityType: null };
|
|
41
|
+
}
|
|
42
|
+
return { tag, entityType: entityType ?? null };
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Decodes BIO-tagged tokens into entity spans
|
|
46
|
+
*/
|
|
47
|
+
export function decodeBIOTags(tokens, labels, confidences, originalText) {
|
|
48
|
+
const entities = [];
|
|
49
|
+
let currentEntity = null;
|
|
50
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
51
|
+
const token = tokens[i];
|
|
52
|
+
const label = labels[i] ?? 'O';
|
|
53
|
+
const confidence = confidences[i] ?? 0;
|
|
54
|
+
// Skip special tokens
|
|
55
|
+
if (token.isSpecial) {
|
|
56
|
+
// If we have a current entity, close it
|
|
57
|
+
if (currentEntity !== null) {
|
|
58
|
+
entities.push(currentEntity);
|
|
59
|
+
currentEntity = null;
|
|
60
|
+
}
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const { tag, entityType } = parseBIOLabel(label);
|
|
64
|
+
switch (tag) {
|
|
65
|
+
case BIOTag.B:
|
|
66
|
+
// Start of new entity
|
|
67
|
+
// Close previous entity if exists
|
|
68
|
+
if (currentEntity !== null) {
|
|
69
|
+
entities.push(currentEntity);
|
|
70
|
+
}
|
|
71
|
+
currentEntity = {
|
|
72
|
+
type: entityType ?? 'UNKNOWN',
|
|
73
|
+
start: token.start,
|
|
74
|
+
end: token.end,
|
|
75
|
+
confidence,
|
|
76
|
+
text: originalText.slice(token.start, token.end),
|
|
77
|
+
tokenIndices: [i],
|
|
78
|
+
};
|
|
79
|
+
break;
|
|
80
|
+
case BIOTag.I:
|
|
81
|
+
// Continuation of entity
|
|
82
|
+
if (currentEntity !== null && entityType === currentEntity.type) {
|
|
83
|
+
// Extend current entity
|
|
84
|
+
currentEntity.end = token.end;
|
|
85
|
+
currentEntity.text = originalText.slice(currentEntity.start, currentEntity.end);
|
|
86
|
+
currentEntity.tokenIndices.push(i);
|
|
87
|
+
// Average confidence
|
|
88
|
+
currentEntity.confidence =
|
|
89
|
+
(currentEntity.confidence * (currentEntity.tokenIndices.length - 1) + confidence) /
|
|
90
|
+
currentEntity.tokenIndices.length;
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
// I tag without matching B tag - treat as new entity (common in some models)
|
|
94
|
+
if (currentEntity !== null) {
|
|
95
|
+
entities.push(currentEntity);
|
|
96
|
+
}
|
|
97
|
+
currentEntity = {
|
|
98
|
+
type: entityType ?? 'UNKNOWN',
|
|
99
|
+
start: token.start,
|
|
100
|
+
end: token.end,
|
|
101
|
+
confidence,
|
|
102
|
+
text: originalText.slice(token.start, token.end),
|
|
103
|
+
tokenIndices: [i],
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
break;
|
|
107
|
+
case BIOTag.O:
|
|
108
|
+
// Outside entity - close current if exists
|
|
109
|
+
if (currentEntity !== null) {
|
|
110
|
+
entities.push(currentEntity);
|
|
111
|
+
currentEntity = null;
|
|
112
|
+
}
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// Don't forget to close the last entity
|
|
117
|
+
if (currentEntity !== null) {
|
|
118
|
+
entities.push(currentEntity);
|
|
119
|
+
}
|
|
120
|
+
return entities;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Converts raw NER entities to SpanMatch format
|
|
124
|
+
*/
|
|
125
|
+
export function convertToSpanMatches(rawEntities, confidenceThreshold = 0.5) {
|
|
126
|
+
const spans = [];
|
|
127
|
+
for (const entity of rawEntities) {
|
|
128
|
+
// Filter by confidence
|
|
129
|
+
if (entity.confidence < confidenceThreshold) {
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
// Map entity type to PIIType
|
|
133
|
+
const piiType = getPIITypeFromNERLabel(entity.type);
|
|
134
|
+
if (piiType === null) {
|
|
135
|
+
continue; // Skip unknown types
|
|
136
|
+
}
|
|
137
|
+
spans.push({
|
|
138
|
+
type: piiType,
|
|
139
|
+
start: entity.start,
|
|
140
|
+
end: entity.end,
|
|
141
|
+
confidence: entity.confidence,
|
|
142
|
+
source: DetectionSource.NER,
|
|
143
|
+
text: entity.text,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return spans;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Post-processes NER spans to clean up boundaries
|
|
150
|
+
*/
|
|
151
|
+
export function cleanupSpanBoundaries(spans, originalText) {
|
|
152
|
+
return spans.map((span) => {
|
|
153
|
+
let { start, end } = span;
|
|
154
|
+
// Trim leading whitespace
|
|
155
|
+
while (start < end && /\s/.test(originalText[start] ?? '')) {
|
|
156
|
+
start++;
|
|
157
|
+
}
|
|
158
|
+
// Trim trailing whitespace
|
|
159
|
+
while (end > start && /\s/.test(originalText[end - 1] ?? '')) {
|
|
160
|
+
end--;
|
|
161
|
+
}
|
|
162
|
+
// Trim leading/trailing punctuation for PERSON/ORG types
|
|
163
|
+
if (span.type === PIIType.PERSON || span.type === PIIType.ORG) {
|
|
164
|
+
while (start < end && /[.,;:!?'"()]/.test(originalText[start] ?? '')) {
|
|
165
|
+
start++;
|
|
166
|
+
}
|
|
167
|
+
while (end > start && /[.,;:!?'"()]/.test(originalText[end - 1] ?? '')) {
|
|
168
|
+
end--;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
// If span became empty, return original
|
|
172
|
+
if (start >= end) {
|
|
173
|
+
return span;
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
...span,
|
|
177
|
+
start,
|
|
178
|
+
end,
|
|
179
|
+
text: originalText.slice(start, end),
|
|
180
|
+
};
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Merges adjacent spans of the same type
|
|
185
|
+
*/
|
|
186
|
+
export function mergeAdjacentSpans(spans, originalText, maxGap = 1) {
|
|
187
|
+
if (spans.length <= 1)
|
|
188
|
+
return spans;
|
|
189
|
+
// Sort by start position
|
|
190
|
+
const sorted = [...spans].sort((a, b) => a.start - b.start);
|
|
191
|
+
const merged = [];
|
|
192
|
+
let current = sorted[0];
|
|
193
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
194
|
+
const next = sorted[i];
|
|
195
|
+
// Check if same type and close enough
|
|
196
|
+
const gap = next.start - current.end;
|
|
197
|
+
const gapText = originalText.slice(current.end, next.start);
|
|
198
|
+
const isOnlyWhitespace = /^\s*$/.test(gapText);
|
|
199
|
+
if (next.type === current.type && gap <= maxGap && isOnlyWhitespace) {
|
|
200
|
+
// Merge spans; max() so a span contained in the current one
|
|
201
|
+
// (possible with overlapping inputs) can never shrink the result
|
|
202
|
+
const mergedEnd = Math.max(current.end, next.end);
|
|
203
|
+
current = {
|
|
204
|
+
...current,
|
|
205
|
+
end: mergedEnd,
|
|
206
|
+
text: originalText.slice(current.start, mergedEnd),
|
|
207
|
+
confidence: (current.confidence + next.confidence) / 2,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
merged.push(current);
|
|
212
|
+
current = next;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
merged.push(current);
|
|
216
|
+
return merged;
|
|
217
|
+
}
|
|
218
|
+
//# sourceMappingURL=bio-decoder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bio-decoder.js","sourceRoot":"","sources":["../../src/ner/bio-decoder.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAa,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAG/D;;GAEG;AACH,MAAM,CAAN,IAAY,MAOX;AAPD,WAAY,MAAM;IAChB,6BAA6B;IAC7B,iBAAO,CAAA;IACP,sCAAsC;IACtC,iBAAO,CAAA;IACP,yBAAyB;IACzB,iBAAO,CAAA;AACT,CAAC,EAPW,MAAM,KAAN,MAAM,QAOjB;AA8BD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACjF,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC;IAEnC,IAAI,GAAW,CAAC;IAChB,QAAQ,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;QAC9B,KAAK,GAAG;YACN,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC;YACf,MAAM;QACR,KAAK,GAAG;YACN,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC;YACf,MAAM;QACR;YACE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;AACjD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAe,EACf,MAAgB,EAChB,WAAqB,EACrB,YAAoB;IAEpB,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,IAAI,aAAa,GAAwB,IAAI,CAAC;IAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QAC/B,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvC,sBAAsB;QACtB,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,wCAAwC;YACxC,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC7B,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAEjD,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,CAAC;gBACX,sBAAsB;gBACtB,kCAAkC;gBAClC,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC/B,CAAC;gBAED,aAAa,GAAG;oBACd,IAAI,EAAE,UAAU,IAAI,SAAS;oBAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU;oBACV,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;oBAChD,YAAY,EAAE,CAAC,CAAC,CAAC;iBAClB,CAAC;gBACF,MAAM;YAER,KAAK,MAAM,CAAC,CAAC;gBACX,yBAAyB;gBACzB,IAAI,aAAa,KAAK,IAAI,IAAI,UAAU,KAAK,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChE,wBAAwB;oBACxB,aAAa,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;oBAC9B,aAAa,CAAC,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC;oBAChF,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBACnC,qBAAqB;oBACrB,aAAa,CAAC,UAAU;wBACtB,CAAC,aAAa,CAAC,UAAU,GAAG,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;4BACjF,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC;gBACtC,CAAC;qBAAM,CAAC;oBACN,6EAA6E;oBAC7E,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;wBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC/B,CAAC;oBACD,aAAa,GAAG;wBACd,IAAI,EAAE,UAAU,IAAI,SAAS;wBAC7B,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;wBACd,UAAU;wBACV,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;wBAChD,YAAY,EAAE,CAAC,CAAC,CAAC;qBAClB,CAAC;gBACJ,CAAC;gBACD,MAAM;YAER,KAAK,MAAM,CAAC,CAAC;gBACX,2CAA2C;gBAC3C,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC7B,aAAa,GAAG,IAAI,CAAC;gBACvB,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,WAA2B,EAC3B,sBAA8B,GAAG;IAEjC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,uBAAuB;QACvB,IAAI,MAAM,CAAC,UAAU,GAAG,mBAAmB,EAAE,CAAC;YAC5C,SAAS;QACX,CAAC;QAED,6BAA6B;QAC7B,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,SAAS,CAAC,qBAAqB;QACjC,CAAC;QAED,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,eAAe,CAAC,GAAG;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAkB,EAClB,YAAoB;IAEpB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAE1B,0BAA0B;QAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC3D,KAAK,EAAE,CAAC;QACV,CAAC;QAED,2BAA2B;QAC3B,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC7D,GAAG,EAAE,CAAC;QACR,CAAC;QAED,yDAAyD;QACzD,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;YAC9D,OAAO,KAAK,GAAG,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACrE,KAAK,EAAE,CAAC;YACV,CAAC;YACD,OAAO,GAAG,GAAG,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBACvE,GAAG,EAAE,CAAC;YACR,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,KAAK;YACL,GAAG;YACH,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;SACrC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAkB,EAClB,YAAoB,EACpB,SAAiB,CAAC;IAElB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpC,yBAAyB;IACzB,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QAExB,sCAAsC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;QACrC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,IAAI,GAAG,IAAI,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpE,4DAA4D;YAC5D,iEAAiE;YACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAClD,OAAO,GAAG;gBACR,GAAG,OAAO;gBACV,GAAG,EAAE,SAAS;gBACd,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC;gBAClD,UAAU,EAAE,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;aACvD,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NER Module
|
|
3
|
+
* Exports NER model and tokenizer components
|
|
4
|
+
*/
|
|
5
|
+
export * from "./tokenizer.js";
|
|
6
|
+
export * from "./bio-decoder.js";
|
|
7
|
+
export * from "./ner-model.js";
|
|
8
|
+
export * from "./inference-server-client.js";
|
|
9
|
+
export * from "./inference-server-ner-model.js";
|
|
10
|
+
export { loadRuntime, detectRuntime, getRuntimeType, type OrtSessionOptions, } from "#onnx-runtime";
|
|
11
|
+
export { type NERModelMode, type ModelInfo, type ModelFileInfo, type DownloadProgressCallback, MODEL_REGISTRY, getModelCacheDir, getModelPath, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, getModelInfo, } from "./model-manager.js";
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ner/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,iCAAiC,CAAC;AAChD,OAAO,EACL,WAAW,EACX,aAAa,EACb,cAAc,EACd,KAAK,iBAAiB,GACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,EACpB,YAAY,GACb,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NER Module
|
|
3
|
+
* Exports NER model and tokenizer components
|
|
4
|
+
*/
|
|
5
|
+
export * from "./tokenizer.js";
|
|
6
|
+
export * from "./bio-decoder.js";
|
|
7
|
+
export * from "./ner-model.js";
|
|
8
|
+
export * from "./inference-server-client.js";
|
|
9
|
+
export * from "./inference-server-ner-model.js";
|
|
10
|
+
export { loadRuntime, detectRuntime, getRuntimeType, } from "#onnx-runtime";
|
|
11
|
+
export { MODEL_REGISTRY, getModelCacheDir, getModelPath, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, getModelInfo, } from "./model-manager.js";
|
|
12
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ner/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,iCAAiC,CAAC;AAChD,OAAO,EACL,WAAW,EACX,aAAa,EACb,cAAc,GAEf,MAAM,eAAe,CAAC;AACvB,OAAO,EAKL,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,EACpB,YAAY,GACb,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inference Server Client
|
|
3
|
+
* HTTP client for GPU-accelerated NER inference
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Client configuration
|
|
7
|
+
*/
|
|
8
|
+
export interface InferenceServerConfig {
|
|
9
|
+
/** Server URL (e.g., 'http://localhost:8080') */
|
|
10
|
+
url: string;
|
|
11
|
+
/** Request timeout in ms (default: 30000) */
|
|
12
|
+
timeout?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Entity match from the server
|
|
16
|
+
*/
|
|
17
|
+
export interface ServerEntityMatch {
|
|
18
|
+
/** Entity type (PERSON, ORG, LOCATION, etc.) */
|
|
19
|
+
type: string;
|
|
20
|
+
/** Start character offset */
|
|
21
|
+
start: number;
|
|
22
|
+
/** End character offset */
|
|
23
|
+
end: number;
|
|
24
|
+
/** Confidence (0-1) */
|
|
25
|
+
confidence: number;
|
|
26
|
+
/** Matched text */
|
|
27
|
+
text: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Health status
|
|
31
|
+
*/
|
|
32
|
+
export interface HealthStatus {
|
|
33
|
+
status: string;
|
|
34
|
+
model_loaded: boolean;
|
|
35
|
+
provider: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Client for the Rehydra NER inference server.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const client = new InferenceServerClient({ url: 'http://localhost:8080' });
|
|
43
|
+
*
|
|
44
|
+
* await client.waitUntilReady();
|
|
45
|
+
*
|
|
46
|
+
* const result = await client.predict('John Smith works at Acme Corp');
|
|
47
|
+
* console.log(result.entities);
|
|
48
|
+
* // [{ type: 'PERSON', text: 'John Smith', ... }, { type: 'ORG', text: 'Acme Corp', ... }]
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
export declare class InferenceServerClient {
|
|
52
|
+
private readonly url;
|
|
53
|
+
private readonly timeout;
|
|
54
|
+
constructor(config: InferenceServerConfig);
|
|
55
|
+
/**
|
|
56
|
+
* Run NER prediction on text
|
|
57
|
+
*
|
|
58
|
+
* @param text - Text to analyze
|
|
59
|
+
* @param confidenceThreshold - Minimum confidence (0-1, default: 0.5)
|
|
60
|
+
*/
|
|
61
|
+
predict(text: string, confidenceThreshold?: number): Promise<{
|
|
62
|
+
entities: ServerEntityMatch[];
|
|
63
|
+
processingTimeMs: number;
|
|
64
|
+
}>;
|
|
65
|
+
/**
|
|
66
|
+
* Check server health
|
|
67
|
+
*/
|
|
68
|
+
health(): Promise<HealthStatus>;
|
|
69
|
+
/**
|
|
70
|
+
* Wait for server to be ready
|
|
71
|
+
*
|
|
72
|
+
* @param maxWaitMs - Max wait time (default: 300000 = 5 min)
|
|
73
|
+
* @param pollIntervalMs - Poll interval (default: 2000)
|
|
74
|
+
*/
|
|
75
|
+
waitUntilReady(maxWaitMs?: number, pollIntervalMs?: number): Promise<HealthStatus>;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Create client instance
|
|
79
|
+
*/
|
|
80
|
+
export declare function createInferenceServerClient(config: InferenceServerConfig): InferenceServerClient;
|
|
81
|
+
//# sourceMappingURL=inference-server-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inference-server-client.d.ts","sourceRoot":"","sources":["../../src/ner/inference-server-client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,iDAAiD;IACjD,GAAG,EAAE,MAAM,CAAC;IACZ,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAUD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,MAAM,EAAE,qBAAqB;IAKzC;;;;;OAKG;IACG,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,mBAAmB,GAAE,MAAY,GAChC,OAAO,CAAC;QAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,CAAC;IAmCvE;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IAyBrC;;;;;OAKG;IACG,cAAc,CAClB,SAAS,SAAS,EAClB,cAAc,SAAO,GACpB,OAAO,CAAC,YAAY,CAAC;CAiBzB;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,qBAAqB,GAC5B,qBAAqB,CAEvB"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inference Server Client
|
|
3
|
+
* HTTP client for GPU-accelerated NER inference
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Client for the Rehydra NER inference server.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* const client = new InferenceServerClient({ url: 'http://localhost:8080' });
|
|
11
|
+
*
|
|
12
|
+
* await client.waitUntilReady();
|
|
13
|
+
*
|
|
14
|
+
* const result = await client.predict('John Smith works at Acme Corp');
|
|
15
|
+
* console.log(result.entities);
|
|
16
|
+
* // [{ type: 'PERSON', text: 'John Smith', ... }, { type: 'ORG', text: 'Acme Corp', ... }]
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export class InferenceServerClient {
|
|
20
|
+
url;
|
|
21
|
+
timeout;
|
|
22
|
+
constructor(config) {
|
|
23
|
+
this.url = config.url.replace(/\/$/, "");
|
|
24
|
+
this.timeout = config.timeout ?? 30000;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Run NER prediction on text
|
|
28
|
+
*
|
|
29
|
+
* @param text - Text to analyze
|
|
30
|
+
* @param confidenceThreshold - Minimum confidence (0-1, default: 0.5)
|
|
31
|
+
*/
|
|
32
|
+
async predict(text, confidenceThreshold = 0.5) {
|
|
33
|
+
const controller = new AbortController();
|
|
34
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
35
|
+
try {
|
|
36
|
+
const response = await fetch(`${this.url}/v1/predict`, {
|
|
37
|
+
method: "POST",
|
|
38
|
+
headers: { "Content-Type": "application/json" },
|
|
39
|
+
body: JSON.stringify({
|
|
40
|
+
text,
|
|
41
|
+
confidence_threshold: confidenceThreshold,
|
|
42
|
+
}),
|
|
43
|
+
signal: controller.signal,
|
|
44
|
+
});
|
|
45
|
+
if (!response.ok) {
|
|
46
|
+
const errorText = await response.text();
|
|
47
|
+
throw new Error(`Server error (${response.status}): ${errorText}`);
|
|
48
|
+
}
|
|
49
|
+
const data = (await response.json());
|
|
50
|
+
return {
|
|
51
|
+
entities: data.entities,
|
|
52
|
+
processingTimeMs: data.processing_time_ms,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
57
|
+
throw new Error(`Request timed out after ${this.timeout}ms`);
|
|
58
|
+
}
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
clearTimeout(timeoutId);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Check server health
|
|
67
|
+
*/
|
|
68
|
+
async health() {
|
|
69
|
+
const controller = new AbortController();
|
|
70
|
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
71
|
+
try {
|
|
72
|
+
const response = await fetch(`${this.url}/health`, {
|
|
73
|
+
method: "GET",
|
|
74
|
+
signal: controller.signal,
|
|
75
|
+
});
|
|
76
|
+
if (!response.ok) {
|
|
77
|
+
throw new Error(`Health check failed (${response.status})`);
|
|
78
|
+
}
|
|
79
|
+
return (await response.json());
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
83
|
+
throw new Error("Health check timed out");
|
|
84
|
+
}
|
|
85
|
+
throw error;
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
clearTimeout(timeoutId);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Wait for server to be ready
|
|
93
|
+
*
|
|
94
|
+
* @param maxWaitMs - Max wait time (default: 300000 = 5 min)
|
|
95
|
+
* @param pollIntervalMs - Poll interval (default: 2000)
|
|
96
|
+
*/
|
|
97
|
+
async waitUntilReady(maxWaitMs = 300000, pollIntervalMs = 2000) {
|
|
98
|
+
const startTime = Date.now();
|
|
99
|
+
while (Date.now() - startTime < maxWaitMs) {
|
|
100
|
+
try {
|
|
101
|
+
const health = await this.health();
|
|
102
|
+
if (health.model_loaded) {
|
|
103
|
+
return health;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
// Not ready yet
|
|
108
|
+
}
|
|
109
|
+
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
|
110
|
+
}
|
|
111
|
+
throw new Error(`Server not ready within ${maxWaitMs}ms`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Create client instance
|
|
116
|
+
*/
|
|
117
|
+
export function createInferenceServerClient(config) {
|
|
118
|
+
return new InferenceServerClient(config);
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=inference-server-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inference-server-client.js","sourceRoot":"","sources":["../../src/ner/inference-server-client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA6CH;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,qBAAqB;IACf,GAAG,CAAS;IACZ,OAAO,CAAS;IAEjC,YAAY,MAA6B;QACvC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IACzC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CACX,IAAY,EACZ,sBAA8B,GAAG;QAEjC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,aAAa,EAAE;gBACrD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI;oBACJ,oBAAoB,EAAE,mBAAmB;iBAC1C,CAAC;gBACF,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAoB,CAAC;YACxD,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,gBAAgB,EAAE,IAAI,CAAC,kBAAkB;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;QAE7D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,SAAS,EAAE;gBACjD,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAC9D,CAAC;YAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAiB,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC5C,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAClB,SAAS,GAAG,MAAM,EAClB,cAAc,GAAG,IAAI;QAErB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACnC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;oBACxB,OAAO,MAAM,CAAC;gBAChB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,gBAAgB;YAClB,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;IAC5D,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,2BAA2B,CACzC,MAA6B;IAE7B,OAAO,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;AAC3C,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inference Server NER Model
|
|
3
|
+
* NER model using remote GPU inference with server-side tokenization
|
|
4
|
+
*/
|
|
5
|
+
import { AnonymizationPolicy } from "../types/index.js";
|
|
6
|
+
import type { INERModel, NERPrediction } from "./ner-model.js";
|
|
7
|
+
/**
|
|
8
|
+
* Configuration
|
|
9
|
+
*/
|
|
10
|
+
export interface InferenceServerNERModelConfig {
|
|
11
|
+
/** Server URL (e.g., 'http://localhost:8080') */
|
|
12
|
+
serverUrl: string;
|
|
13
|
+
/** Request timeout in ms (default: 30000) */
|
|
14
|
+
timeout?: number;
|
|
15
|
+
/** Model version string */
|
|
16
|
+
modelVersion?: string;
|
|
17
|
+
/** Status callback */
|
|
18
|
+
onStatus?: (message: string) => void;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* NER Model that delegates to a GPU inference server.
|
|
22
|
+
*
|
|
23
|
+
* Server handles: tokenization, inference, BIO decoding, span cleanup
|
|
24
|
+
* Client handles: type mapping, policy filtering
|
|
25
|
+
*/
|
|
26
|
+
export declare class InferenceServerNERModel implements INERModel {
|
|
27
|
+
private client;
|
|
28
|
+
private config;
|
|
29
|
+
private isLoaded;
|
|
30
|
+
readonly version: string;
|
|
31
|
+
constructor(config: InferenceServerNERModelConfig);
|
|
32
|
+
get loaded(): boolean;
|
|
33
|
+
load(): Promise<void>;
|
|
34
|
+
predict(text: string, policy?: AnonymizationPolicy): Promise<NERPrediction>;
|
|
35
|
+
private convertToSpanMatches;
|
|
36
|
+
private getMinConfidence;
|
|
37
|
+
dispose(): Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Create an Inference Server NER model
|
|
41
|
+
*/
|
|
42
|
+
export declare function createInferenceServerNERModel(config: InferenceServerNERModelConfig): InferenceServerNERModel;
|
|
43
|
+
//# sourceMappingURL=inference-server-ner-model.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inference-server-ner-model.d.ts","sourceRoot":"","sources":["../../src/ner/inference-server-ner-model.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAa,mBAAmB,EAA4B,MAAM,mBAAmB,CAAC;AAK7F,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC5C,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sBAAsB;IACtB,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAaD;;;;;GAKG;AACH,qBAAa,uBAAwB,YAAW,SAAS;IACvD,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,QAAQ,CAAS;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;gBAEb,MAAM,EAAE,6BAA6B;IASjD,IAAI,MAAM,IAAI,OAAO,CAEpB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBrB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC;IA8BjF,OAAO,CAAC,oBAAoB;IAoB5B,OAAO,CAAC,gBAAgB;IAcxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAIzB;AAED;;GAEG;AACH,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,6BAA6B,GACpC,uBAAuB,CAEzB"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inference Server NER Model
|
|
3
|
+
* NER model using remote GPU inference with server-side tokenization
|
|
4
|
+
*/
|
|
5
|
+
import { DetectionSource, PIIType } from "../types/index.js";
|
|
6
|
+
import { InferenceServerClient, } from "./inference-server-client.js";
|
|
7
|
+
// Server type → SDK PIIType mapping
|
|
8
|
+
const SERVER_TYPE_TO_PII_TYPE = {
|
|
9
|
+
'PERSON': PIIType.PERSON,
|
|
10
|
+
'PER': PIIType.PERSON,
|
|
11
|
+
'ORG': PIIType.ORG,
|
|
12
|
+
'ORGANIZATION': PIIType.ORG,
|
|
13
|
+
'LOCATION': PIIType.LOCATION,
|
|
14
|
+
'LOC': PIIType.LOCATION,
|
|
15
|
+
'GPE': PIIType.LOCATION,
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* NER Model that delegates to a GPU inference server.
|
|
19
|
+
*
|
|
20
|
+
* Server handles: tokenization, inference, BIO decoding, span cleanup
|
|
21
|
+
* Client handles: type mapping, policy filtering
|
|
22
|
+
*/
|
|
23
|
+
export class InferenceServerNERModel {
|
|
24
|
+
client;
|
|
25
|
+
config;
|
|
26
|
+
isLoaded = false;
|
|
27
|
+
version;
|
|
28
|
+
constructor(config) {
|
|
29
|
+
this.config = config;
|
|
30
|
+
this.version = config.modelVersion ?? "2.0.0";
|
|
31
|
+
this.client = new InferenceServerClient({
|
|
32
|
+
url: config.serverUrl,
|
|
33
|
+
timeout: config.timeout,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
get loaded() {
|
|
37
|
+
return this.isLoaded;
|
|
38
|
+
}
|
|
39
|
+
async load() {
|
|
40
|
+
if (this.isLoaded)
|
|
41
|
+
return;
|
|
42
|
+
this.config.onStatus?.("Connecting to inference server...");
|
|
43
|
+
try {
|
|
44
|
+
const health = await this.client.health();
|
|
45
|
+
if (!health.model_loaded) {
|
|
46
|
+
this.config.onStatus?.("Waiting for server to load model...");
|
|
47
|
+
await this.client.waitUntilReady();
|
|
48
|
+
}
|
|
49
|
+
this.config.onStatus?.(`Connected (${health.provider})`);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
throw new Error(`Failed to connect to ${this.config.serverUrl}: ${error instanceof Error ? error.message : String(error)}`);
|
|
53
|
+
}
|
|
54
|
+
this.isLoaded = true;
|
|
55
|
+
this.config.onStatus?.("Ready");
|
|
56
|
+
}
|
|
57
|
+
async predict(text, policy) {
|
|
58
|
+
if (!this.isLoaded) {
|
|
59
|
+
throw new Error("Model not loaded. Call load() first.");
|
|
60
|
+
}
|
|
61
|
+
const minConfidence = this.getMinConfidence(policy);
|
|
62
|
+
const result = await this.client.predict(text, minConfidence);
|
|
63
|
+
let spans = this.convertToSpanMatches(result.entities);
|
|
64
|
+
// Apply policy filtering
|
|
65
|
+
if (policy !== undefined) {
|
|
66
|
+
spans = spans.filter((span) => policy.enabledTypes.has(span.type) &&
|
|
67
|
+
policy.nerEnabledTypes.has(span.type));
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
spans,
|
|
71
|
+
processingTimeMs: result.processingTimeMs,
|
|
72
|
+
modelVersion: this.version,
|
|
73
|
+
// The remote server owns tokenization; windowing/capping does not apply
|
|
74
|
+
// on this path, so coverage is never client-side truncated.
|
|
75
|
+
truncated: false,
|
|
76
|
+
coverageChars: text.length,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
convertToSpanMatches(entities) {
|
|
80
|
+
const spans = [];
|
|
81
|
+
for (const entity of entities) {
|
|
82
|
+
const piiType = SERVER_TYPE_TO_PII_TYPE[entity.type.toUpperCase()];
|
|
83
|
+
if (piiType === undefined)
|
|
84
|
+
continue;
|
|
85
|
+
spans.push({
|
|
86
|
+
type: piiType,
|
|
87
|
+
start: entity.start,
|
|
88
|
+
end: entity.end,
|
|
89
|
+
confidence: entity.confidence,
|
|
90
|
+
source: DetectionSource.NER,
|
|
91
|
+
text: entity.text,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
return spans;
|
|
95
|
+
}
|
|
96
|
+
getMinConfidence(policy) {
|
|
97
|
+
if (policy === undefined)
|
|
98
|
+
return 0.5;
|
|
99
|
+
let minThreshold = 1.0;
|
|
100
|
+
for (const type of policy.nerEnabledTypes) {
|
|
101
|
+
const threshold = policy.confidenceThresholds.get(type) ?? 0.5;
|
|
102
|
+
if (threshold < minThreshold) {
|
|
103
|
+
minThreshold = threshold;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return minThreshold;
|
|
107
|
+
}
|
|
108
|
+
dispose() {
|
|
109
|
+
this.isLoaded = false;
|
|
110
|
+
return Promise.resolve();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Create an Inference Server NER model
|
|
115
|
+
*/
|
|
116
|
+
export function createInferenceServerNERModel(config) {
|
|
117
|
+
return new InferenceServerNERModel(config);
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=inference-server-ner-model.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inference-server-ner-model.js","sourceRoot":"","sources":["../../src/ner/inference-server-ner-model.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAkC,eAAe,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EACL,qBAAqB,GAEtB,MAAM,8BAA8B,CAAC;AAiBtC,oCAAoC;AACpC,MAAM,uBAAuB,GAA4B;IACvD,QAAQ,EAAE,OAAO,CAAC,MAAM;IACxB,KAAK,EAAE,OAAO,CAAC,MAAM;IACrB,KAAK,EAAE,OAAO,CAAC,GAAG;IAClB,cAAc,EAAE,OAAO,CAAC,GAAG;IAC3B,UAAU,EAAE,OAAO,CAAC,QAAQ;IAC5B,KAAK,EAAE,OAAO,CAAC,QAAQ;IACvB,KAAK,EAAE,OAAO,CAAC,QAAQ;CACxB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,uBAAuB;IAC1B,MAAM,CAAwB;IAC9B,MAAM,CAAgC;IACtC,QAAQ,GAAG,KAAK,CAAC;IAChB,OAAO,CAAS;IAEzB,YAAY,MAAqC;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,qBAAqB,CAAC;YACtC,GAAG,EAAE,MAAM,CAAC,SAAS;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,mCAAmC,CAAC,CAAC;QAE5D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,qCAAqC,CAAC,CAAC;gBAC9D,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACrC,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,cAAc,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,wBAAwB,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC3G,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAY,EAAE,MAA4B;QACtD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAE9D,IAAI,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEvD,yBAAyB;QACzB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,IAAI,EAAE,EAAE,CACP,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACxC,CAAC;QACJ,CAAC;QAED,OAAO;YACL,KAAK;YACL,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,YAAY,EAAE,IAAI,CAAC,OAAO;YAC1B,wEAAwE;YACxE,4DAA4D;YAC5D,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,IAAI,CAAC,MAAM;SAC3B,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,QAA6B;QACxD,MAAM,KAAK,GAAgB,EAAE,CAAC;QAE9B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnE,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAS;YAEpC,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,MAAM,EAAE,eAAe,CAAC,GAAG;gBAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,gBAAgB,CAAC,MAA4B;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAErC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YAC/D,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;gBAC7B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,OAAO;QACL,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,6BAA6B,CAC3C,MAAqC;IAErC,OAAO,IAAI,uBAAuB,CAAC,MAAM,CAAC,CAAC;AAC7C,CAAC"}
|