opencode-rehydar 0.14.2 → 0.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +54 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +74 -0
- package/dist/browser.js.map +1 -0
- package/dist/core/anonymizer.d.ts +309 -0
- package/dist/core/anonymizer.d.ts.map +1 -0
- package/dist/core/anonymizer.js +555 -0
- package/dist/core/anonymizer.js.map +1 -0
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +6 -0
- package/dist/core/index.js.map +1 -0
- package/dist/crypto/index.d.ts +6 -0
- package/dist/crypto/index.d.ts.map +1 -0
- package/dist/crypto/index.js +6 -0
- package/dist/crypto/index.js.map +1 -0
- package/dist/crypto/pii-map-crypto.d.ts +114 -0
- package/dist/crypto/pii-map-crypto.d.ts.map +1 -0
- package/dist/crypto/pii-map-crypto.js +228 -0
- package/dist/crypto/pii-map-crypto.js.map +1 -0
- package/dist/index.d.ts +53 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +72 -0
- package/dist/index.js.map +1 -0
- package/dist/ner/bio-decoder.d.ts +64 -0
- package/dist/ner/bio-decoder.d.ts.map +1 -0
- package/dist/ner/bio-decoder.js +218 -0
- package/dist/ner/bio-decoder.js.map +1 -0
- package/dist/ner/index.d.ts +12 -0
- package/dist/ner/index.d.ts.map +1 -0
- package/dist/ner/index.js +12 -0
- package/dist/ner/index.js.map +1 -0
- package/dist/ner/inference-server-client.d.ts +81 -0
- package/dist/ner/inference-server-client.d.ts.map +1 -0
- package/dist/ner/inference-server-client.js +120 -0
- package/dist/ner/inference-server-client.js.map +1 -0
- package/dist/ner/inference-server-ner-model.d.ts +43 -0
- package/dist/ner/inference-server-ner-model.d.ts.map +1 -0
- package/dist/ner/inference-server-ner-model.js +119 -0
- package/dist/ner/inference-server-ner-model.js.map +1 -0
- package/dist/ner/model-manager.d.ts +111 -0
- package/dist/ner/model-manager.d.ts.map +1 -0
- package/dist/ner/model-manager.js +334 -0
- package/dist/ner/model-manager.js.map +1 -0
- package/dist/ner/ner-model.d.ts +172 -0
- package/dist/ner/ner-model.d.ts.map +1 -0
- package/dist/ner/ner-model.js +514 -0
- package/dist/ner/ner-model.js.map +1 -0
- package/dist/ner/onnx-runtime.browser.d.ts +46 -0
- package/dist/ner/onnx-runtime.browser.d.ts.map +1 -0
- package/dist/ner/onnx-runtime.browser.js +90 -0
- package/dist/ner/onnx-runtime.browser.js.map +1 -0
- package/dist/ner/onnx-runtime.d.ts +72 -0
- package/dist/ner/onnx-runtime.d.ts.map +1 -0
- package/dist/ner/onnx-runtime.js +132 -0
- package/dist/ner/onnx-runtime.js.map +1 -0
- package/dist/ner/token-window.d.ts +19 -0
- package/dist/ner/token-window.d.ts.map +1 -0
- package/dist/ner/token-window.js +101 -0
- package/dist/ner/token-window.js.map +1 -0
- package/dist/ner/tokenizer.d.ts +142 -0
- package/dist/ner/tokenizer.d.ts.map +1 -0
- package/dist/ner/tokenizer.js +362 -0
- package/dist/ner/tokenizer.js.map +1 -0
- package/dist/opencode-plugin/git-identity.d.ts +3 -0
- package/dist/opencode-plugin/git-identity.d.ts.map +1 -0
- package/dist/opencode-plugin/git-identity.js +37 -0
- package/dist/opencode-plugin/git-identity.js.map +1 -0
- package/dist/opencode-plugin/github-username.d.ts +3 -0
- package/dist/opencode-plugin/github-username.d.ts.map +1 -0
- package/dist/opencode-plugin/github-username.js +41 -0
- package/dist/opencode-plugin/github-username.js.map +1 -0
- package/dist/opencode-plugin/identity-recognizer.d.ts +5 -0
- package/dist/opencode-plugin/identity-recognizer.d.ts.map +1 -0
- package/dist/opencode-plugin/identity-recognizer.js +21 -0
- package/dist/opencode-plugin/identity-recognizer.js.map +1 -0
- package/dist/opencode-plugin/index.d.ts +7 -0
- package/dist/opencode-plugin/index.d.ts.map +1 -0
- package/dist/opencode-plugin/index.js +6 -0
- package/dist/opencode-plugin/index.js.map +1 -0
- package/dist/opencode-plugin/plugin.d.ts +39 -0
- package/dist/opencode-plugin/plugin.d.ts.map +1 -0
- package/dist/opencode-plugin/plugin.js +294 -0
- package/dist/opencode-plugin/plugin.js.map +1 -0
- package/dist/opencode-plugin/types.d.ts +33 -0
- package/dist/opencode-plugin/types.d.ts.map +1 -0
- package/dist/opencode-plugin/types.js +5 -0
- package/dist/opencode-plugin/types.js.map +1 -0
- package/dist/opencode-plugin/v2/anon-state.d.ts +12 -0
- package/dist/opencode-plugin/v2/anon-state.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/anon-state.js +14 -0
- package/dist/opencode-plugin/v2/anon-state.js.map +1 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.d.ts +13 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.js +33 -0
- package/dist/opencode-plugin/v2/deep-rehydrate.js.map +1 -0
- package/dist/opencode-plugin/v2/index.d.ts +17 -0
- package/dist/opencode-plugin/v2/index.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/index.js +17 -0
- package/dist/opencode-plugin/v2/index.js.map +1 -0
- package/dist/opencode-plugin/v2/logger.d.ts +24 -0
- package/dist/opencode-plugin/v2/logger.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/logger.js +73 -0
- package/dist/opencode-plugin/v2/logger.js.map +1 -0
- package/dist/opencode-plugin/v2/options.d.ts +63 -0
- package/dist/opencode-plugin/v2/options.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/options.js +183 -0
- package/dist/opencode-plugin/v2/options.js.map +1 -0
- package/dist/opencode-plugin/v2/register-hooks.d.ts +43 -0
- package/dist/opencode-plugin/v2/register-hooks.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/register-hooks.js +160 -0
- package/dist/opencode-plugin/v2/register-hooks.js.map +1 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.d.ts +101 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.js +147 -0
- package/dist/opencode-plugin/v2/rehydrate-tail.js.map +1 -0
- package/dist/opencode-plugin/v2/response-rehydrate.d.ts +78 -0
- package/dist/opencode-plugin/v2/response-rehydrate.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/response-rehydrate.js +210 -0
- package/dist/opencode-plugin/v2/response-rehydrate.js.map +1 -0
- package/dist/opencode-plugin/v2/scrub-messages.d.ts +39 -0
- package/dist/opencode-plugin/v2/scrub-messages.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/scrub-messages.js +260 -0
- package/dist/opencode-plugin/v2/scrub-messages.js.map +1 -0
- package/dist/opencode-plugin/v2/sessions.d.ts +17 -0
- package/dist/opencode-plugin/v2/sessions.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/sessions.js +23 -0
- package/dist/opencode-plugin/v2/sessions.js.map +1 -0
- package/dist/opencode-plugin/v2/sse.d.ts +101 -0
- package/dist/opencode-plugin/v2/sse.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/sse.js +235 -0
- package/dist/opencode-plugin/v2/sse.js.map +1 -0
- package/dist/opencode-plugin/v2/types.d.ts +188 -0
- package/dist/opencode-plugin/v2/types.d.ts.map +1 -0
- package/dist/opencode-plugin/v2/types.js +10 -0
- package/dist/opencode-plugin/v2/types.js.map +1 -0
- package/dist/opencode-plugin/vcs-command.d.ts +3 -0
- package/dist/opencode-plugin/vcs-command.d.ts.map +1 -0
- package/dist/opencode-plugin/vcs-command.js +34 -0
- package/dist/opencode-plugin/vcs-command.js.map +1 -0
- package/dist/pipeline/index.d.ts +12 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/pipeline/index.js +12 -0
- package/dist/pipeline/index.js.map +1 -0
- package/dist/pipeline/prenormalize.d.ts +48 -0
- package/dist/pipeline/prenormalize.d.ts.map +1 -0
- package/dist/pipeline/prenormalize.js +94 -0
- package/dist/pipeline/prenormalize.js.map +1 -0
- package/dist/pipeline/resolver.d.ts +56 -0
- package/dist/pipeline/resolver.d.ts.map +1 -0
- package/dist/pipeline/resolver.js +239 -0
- package/dist/pipeline/resolver.js.map +1 -0
- package/dist/pipeline/semantic-data-loader.d.ts +171 -0
- package/dist/pipeline/semantic-data-loader.d.ts.map +1 -0
- package/dist/pipeline/semantic-data-loader.js +677 -0
- package/dist/pipeline/semantic-data-loader.js.map +1 -0
- package/dist/pipeline/semantic-enricher.d.ts +112 -0
- package/dist/pipeline/semantic-enricher.d.ts.map +1 -0
- package/dist/pipeline/semantic-enricher.js +322 -0
- package/dist/pipeline/semantic-enricher.js.map +1 -0
- package/dist/pipeline/tagger.d.ts +121 -0
- package/dist/pipeline/tagger.d.ts.map +1 -0
- package/dist/pipeline/tagger.js +545 -0
- package/dist/pipeline/tagger.js.map +1 -0
- package/dist/pipeline/title-extractor.d.ts +79 -0
- package/dist/pipeline/title-extractor.d.ts.map +1 -0
- package/dist/pipeline/title-extractor.js +801 -0
- package/dist/pipeline/title-extractor.js.map +1 -0
- package/dist/pipeline/validator.d.ts +65 -0
- package/dist/pipeline/validator.d.ts.map +1 -0
- package/dist/pipeline/validator.js +292 -0
- package/dist/pipeline/validator.js.map +1 -0
- package/dist/proxy/index.d.ts +13 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +12 -0
- package/dist/proxy/index.js.map +1 -0
- package/dist/proxy/providers/anthropic.d.ts +28 -0
- package/dist/proxy/providers/anthropic.d.ts.map +1 -0
- package/dist/proxy/providers/anthropic.js +265 -0
- package/dist/proxy/providers/anthropic.js.map +1 -0
- package/dist/proxy/providers/index.d.ts +19 -0
- package/dist/proxy/providers/index.d.ts.map +1 -0
- package/dist/proxy/providers/index.js +41 -0
- package/dist/proxy/providers/index.js.map +1 -0
- package/dist/proxy/providers/openai.d.ts +27 -0
- package/dist/proxy/providers/openai.d.ts.map +1 -0
- package/dist/proxy/providers/openai.js +203 -0
- package/dist/proxy/providers/openai.js.map +1 -0
- package/dist/proxy/providers/streaming-tool-loop.d.ts +12 -0
- package/dist/proxy/providers/streaming-tool-loop.d.ts.map +1 -0
- package/dist/proxy/providers/streaming-tool-loop.js +223 -0
- package/dist/proxy/providers/streaming-tool-loop.js.map +1 -0
- package/dist/proxy/providers/types.d.ts +109 -0
- package/dist/proxy/providers/types.d.ts.map +1 -0
- package/dist/proxy/providers/types.js +2 -0
- package/dist/proxy/providers/types.js.map +1 -0
- package/dist/proxy/proxy-server.d.ts +61 -0
- package/dist/proxy/proxy-server.d.ts.map +1 -0
- package/dist/proxy/proxy-server.js +146 -0
- package/dist/proxy/proxy-server.js.map +1 -0
- package/dist/proxy/rehydra-fetch.d.ts +35 -0
- package/dist/proxy/rehydra-fetch.d.ts.map +1 -0
- package/dist/proxy/rehydra-fetch.js +631 -0
- package/dist/proxy/rehydra-fetch.js.map +1 -0
- package/dist/proxy/rehydra-proxy.d.ts +40 -0
- package/dist/proxy/rehydra-proxy.d.ts.map +1 -0
- package/dist/proxy/rehydra-proxy.js +150 -0
- package/dist/proxy/rehydra-proxy.js.map +1 -0
- package/dist/proxy/sse-parser.d.ts +59 -0
- package/dist/proxy/sse-parser.d.ts.map +1 -0
- package/dist/proxy/sse-parser.js +112 -0
- package/dist/proxy/sse-parser.js.map +1 -0
- package/dist/proxy/system-instruction.d.ts +19 -0
- package/dist/proxy/system-instruction.d.ts.map +1 -0
- package/dist/proxy/system-instruction.js +33 -0
- package/dist/proxy/system-instruction.js.map +1 -0
- package/dist/proxy/tool-loop-stream.d.ts +7 -0
- package/dist/proxy/tool-loop-stream.d.ts.map +1 -0
- package/dist/proxy/tool-loop-stream.js +65 -0
- package/dist/proxy/tool-loop-stream.js.map +1 -0
- package/dist/proxy/types.d.ts +138 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +5 -0
- package/dist/proxy/types.js.map +1 -0
- package/dist/proxy/wrap-client.d.ts +47 -0
- package/dist/proxy/wrap-client.d.ts.map +1 -0
- package/dist/proxy/wrap-client.js +70 -0
- package/dist/proxy/wrap-client.js.map +1 -0
- package/dist/recognizers/address.d.ts +2 -0
- package/dist/recognizers/address.d.ts.map +1 -0
- package/dist/recognizers/address.js +23 -0
- package/dist/recognizers/address.js.map +1 -0
- package/dist/recognizers/base.d.ts +78 -0
- package/dist/recognizers/base.d.ts.map +1 -0
- package/dist/recognizers/base.js +100 -0
- package/dist/recognizers/base.js.map +1 -0
- package/dist/recognizers/bic-swift.d.ts +10 -0
- package/dist/recognizers/bic-swift.d.ts.map +1 -0
- package/dist/recognizers/bic-swift.js +107 -0
- package/dist/recognizers/bic-swift.js.map +1 -0
- package/dist/recognizers/credit-card.d.ts +32 -0
- package/dist/recognizers/credit-card.d.ts.map +1 -0
- package/dist/recognizers/credit-card.js +160 -0
- package/dist/recognizers/credit-card.js.map +1 -0
- package/dist/recognizers/custom-id.d.ts +28 -0
- package/dist/recognizers/custom-id.d.ts.map +1 -0
- package/dist/recognizers/custom-id.js +116 -0
- package/dist/recognizers/custom-id.js.map +1 -0
- package/dist/recognizers/date.d.ts +10 -0
- package/dist/recognizers/date.d.ts.map +1 -0
- package/dist/recognizers/date.js +228 -0
- package/dist/recognizers/date.js.map +1 -0
- package/dist/recognizers/email.d.ts +10 -0
- package/dist/recognizers/email.d.ts.map +1 -0
- package/dist/recognizers/email.js +75 -0
- package/dist/recognizers/email.js.map +1 -0
- package/dist/recognizers/iban.d.ts +14 -0
- package/dist/recognizers/iban.d.ts.map +1 -0
- package/dist/recognizers/iban.js +67 -0
- package/dist/recognizers/iban.js.map +1 -0
- package/dist/recognizers/index.d.ts +25 -0
- package/dist/recognizers/index.d.ts.map +1 -0
- package/dist/recognizers/index.js +53 -0
- package/dist/recognizers/index.js.map +1 -0
- package/dist/recognizers/ip-address.d.ts +14 -0
- package/dist/recognizers/ip-address.d.ts.map +1 -0
- package/dist/recognizers/ip-address.js +183 -0
- package/dist/recognizers/ip-address.js.map +1 -0
- package/dist/recognizers/phone.d.ts +10 -0
- package/dist/recognizers/phone.d.ts.map +1 -0
- package/dist/recognizers/phone.js +145 -0
- package/dist/recognizers/phone.js.map +1 -0
- package/dist/recognizers/postal-code.d.ts +16 -0
- package/dist/recognizers/postal-code.d.ts.map +1 -0
- package/dist/recognizers/postal-code.js +42 -0
- package/dist/recognizers/postal-code.js.map +1 -0
- package/dist/recognizers/registry.d.ts +59 -0
- package/dist/recognizers/registry.d.ts.map +1 -0
- package/dist/recognizers/registry.js +113 -0
- package/dist/recognizers/registry.js.map +1 -0
- package/dist/recognizers/secrets/api-key.d.ts +6 -0
- package/dist/recognizers/secrets/api-key.d.ts.map +1 -0
- package/dist/recognizers/secrets/api-key.js +40 -0
- package/dist/recognizers/secrets/api-key.js.map +1 -0
- package/dist/recognizers/secrets/aws-credentials.d.ts +7 -0
- package/dist/recognizers/secrets/aws-credentials.d.ts.map +1 -0
- package/dist/recognizers/secrets/aws-credentials.js +68 -0
- package/dist/recognizers/secrets/aws-credentials.js.map +1 -0
- package/dist/recognizers/secrets/config-secret.d.ts +8 -0
- package/dist/recognizers/secrets/config-secret.d.ts.map +1 -0
- package/dist/recognizers/secrets/config-secret.js +59 -0
- package/dist/recognizers/secrets/config-secret.js.map +1 -0
- package/dist/recognizers/secrets/connection-string.d.ts +6 -0
- package/dist/recognizers/secrets/connection-string.d.ts.map +1 -0
- package/dist/recognizers/secrets/connection-string.js +40 -0
- package/dist/recognizers/secrets/connection-string.js.map +1 -0
- package/dist/recognizers/secrets/env-var.d.ts +8 -0
- package/dist/recognizers/secrets/env-var.d.ts.map +1 -0
- package/dist/recognizers/secrets/env-var.js +54 -0
- package/dist/recognizers/secrets/env-var.js.map +1 -0
- package/dist/recognizers/secrets/index.d.ts +22 -0
- package/dist/recognizers/secrets/index.d.ts.map +1 -0
- package/dist/recognizers/secrets/index.js +41 -0
- package/dist/recognizers/secrets/index.js.map +1 -0
- package/dist/recognizers/secrets/jwt.d.ts +6 -0
- package/dist/recognizers/secrets/jwt.d.ts.map +1 -0
- package/dist/recognizers/secrets/jwt.js +47 -0
- package/dist/recognizers/secrets/jwt.js.map +1 -0
- package/dist/recognizers/secrets/key-patterns.d.ts +10 -0
- package/dist/recognizers/secrets/key-patterns.d.ts.map +1 -0
- package/dist/recognizers/secrets/key-patterns.js +15 -0
- package/dist/recognizers/secrets/key-patterns.js.map +1 -0
- package/dist/recognizers/secrets/literal-value.d.ts +10 -0
- package/dist/recognizers/secrets/literal-value.d.ts.map +1 -0
- package/dist/recognizers/secrets/literal-value.js +48 -0
- package/dist/recognizers/secrets/literal-value.js.map +1 -0
- package/dist/recognizers/secrets/private-key.d.ts +6 -0
- package/dist/recognizers/secrets/private-key.d.ts.map +1 -0
- package/dist/recognizers/secrets/private-key.js +16 -0
- package/dist/recognizers/secrets/private-key.js.map +1 -0
- package/dist/recognizers/url.d.ts +14 -0
- package/dist/recognizers/url.d.ts.map +1 -0
- package/dist/recognizers/url.js +121 -0
- package/dist/recognizers/url.js.map +1 -0
- package/dist/storage/browser.d.ts +13 -0
- package/dist/storage/browser.d.ts.map +1 -0
- package/dist/storage/browser.js +13 -0
- package/dist/storage/browser.js.map +1 -0
- package/dist/storage/in-memory.d.ts +60 -0
- package/dist/storage/in-memory.d.ts.map +1 -0
- package/dist/storage/in-memory.js +108 -0
- package/dist/storage/in-memory.js.map +1 -0
- package/dist/storage/index.d.ts +11 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +11 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +69 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +274 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/session-base.d.ts +34 -0
- package/dist/storage/session-base.d.ts.map +1 -0
- package/dist/storage/session-base.js +122 -0
- package/dist/storage/session-base.js.map +1 -0
- package/dist/storage/session.d.ts +19 -0
- package/dist/storage/session.d.ts.map +1 -0
- package/dist/storage/session.js +31 -0
- package/dist/storage/session.js.map +1 -0
- package/dist/storage/sqlite.d.ts +80 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +255 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/storage/types.d.ts +208 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +6 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/streaming/anonymizer-stream.d.ts +63 -0
- package/dist/streaming/anonymizer-stream.d.ts.map +1 -0
- package/dist/streaming/anonymizer-stream.js +184 -0
- package/dist/streaming/anonymizer-stream.js.map +1 -0
- package/dist/streaming/index.d.ts +9 -0
- package/dist/streaming/index.d.ts.map +1 -0
- package/dist/streaming/index.js +8 -0
- package/dist/streaming/index.js.map +1 -0
- package/dist/streaming/sentence-buffer.d.ts +78 -0
- package/dist/streaming/sentence-buffer.d.ts.map +1 -0
- package/dist/streaming/sentence-buffer.js +239 -0
- package/dist/streaming/sentence-buffer.js.map +1 -0
- package/dist/streaming/stream-factory.d.ts +38 -0
- package/dist/streaming/stream-factory.d.ts.map +1 -0
- package/dist/streaming/stream-factory.js +69 -0
- package/dist/streaming/stream-factory.js.map +1 -0
- package/dist/streaming/types.d.ts +121 -0
- package/dist/streaming/types.d.ts.map +1 -0
- package/dist/streaming/types.js +5 -0
- package/dist/streaming/types.js.map +1 -0
- package/dist/types/index.d.ts +269 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +95 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/pii-types.d.ts +64 -0
- package/dist/types/pii-types.d.ts.map +1 -0
- package/dist/types/pii-types.js +153 -0
- package/dist/types/pii-types.js.map +1 -0
- package/dist/utils/iban-checksum.d.ts +23 -0
- package/dist/utils/iban-checksum.d.ts.map +1 -0
- package/dist/utils/iban-checksum.js +106 -0
- package/dist/utils/iban-checksum.js.map +1 -0
- package/dist/utils/index.d.ts +11 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +11 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/json-walk.d.ts +21 -0
- package/dist/utils/json-walk.d.ts.map +1 -0
- package/dist/utils/json-walk.js +44 -0
- package/dist/utils/json-walk.js.map +1 -0
- package/dist/utils/luhn.d.ts +17 -0
- package/dist/utils/luhn.d.ts.map +1 -0
- package/dist/utils/luhn.js +55 -0
- package/dist/utils/luhn.js.map +1 -0
- package/dist/utils/offsets.d.ts +86 -0
- package/dist/utils/offsets.d.ts.map +1 -0
- package/dist/utils/offsets.js +124 -0
- package/dist/utils/offsets.js.map +1 -0
- package/dist/utils/path.d.ts +34 -0
- package/dist/utils/path.d.ts.map +1 -0
- package/dist/utils/path.js +96 -0
- package/dist/utils/path.js.map +1 -0
- package/dist/utils/regex.d.ts +11 -0
- package/dist/utils/regex.d.ts.map +1 -0
- package/dist/utils/regex.js +14 -0
- package/dist/utils/regex.js.map +1 -0
- package/dist/utils/storage-browser.d.ts +51 -0
- package/dist/utils/storage-browser.d.ts.map +1 -0
- package/dist/utils/storage-browser.js +381 -0
- package/dist/utils/storage-browser.js.map +1 -0
- package/dist/utils/storage-node.d.ts +51 -0
- package/dist/utils/storage-node.d.ts.map +1 -0
- package/dist/utils/storage-node.js +153 -0
- package/dist/utils/storage-node.js.map +1 -0
- package/dist/utils/storage.browser.d.ts +75 -0
- package/dist/utils/storage.browser.d.ts.map +1 -0
- package/dist/utils/storage.browser.js +69 -0
- package/dist/utils/storage.browser.js.map +1 -0
- package/dist/utils/storage.d.ts +73 -0
- package/dist/utils/storage.d.ts.map +1 -0
- package/dist/utils/storage.js +69 -0
- package/dist/utils/storage.js.map +1 -0
- package/index.js +6 -4
- package/package.json +19 -5
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ONNX Runtime Abstraction - Browser-Only Version
|
|
3
|
+
*
|
|
4
|
+
* This module provides the same interface as onnx-runtime.ts but only includes
|
|
5
|
+
* onnxruntime-web. It's used by the browser entry point to avoid bundler issues
|
|
6
|
+
* with onnxruntime-node.
|
|
7
|
+
*
|
|
8
|
+
* DO NOT import onnxruntime-node here - that's the whole point!
|
|
9
|
+
*/
|
|
10
|
+
// CDN URL for onnxruntime-web (used when bare import fails in browser)
|
|
11
|
+
// Using the bundled ESM version that includes WebAssembly backend
|
|
12
|
+
const ONNX_WEB_CDN_URL = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/ort.bundle.min.mjs";
|
|
13
|
+
/**
|
|
14
|
+
* Runtime detection and loading
|
|
15
|
+
*/
|
|
16
|
+
let _runtime = null;
|
|
17
|
+
let _runtimeType = null;
|
|
18
|
+
/**
|
|
19
|
+
* Detects the best ONNX runtime for the current environment
|
|
20
|
+
* Browser version always returns "web"
|
|
21
|
+
*/
|
|
22
|
+
export function detectRuntime() {
|
|
23
|
+
return "web";
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Attempts to load onnxruntime-web, first via bare import, then via CDN
|
|
27
|
+
*/
|
|
28
|
+
async function loadOnnxWeb() {
|
|
29
|
+
// First try bare import (works with bundlers or import maps)
|
|
30
|
+
try {
|
|
31
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
32
|
+
// @ts-ignore - onnxruntime-web may not be installed
|
|
33
|
+
const ort = (await import("onnxruntime-web"));
|
|
34
|
+
return ort;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// Bare import failed, try CDN (for browsers without bundlers)
|
|
38
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
39
|
+
const isBrowser = typeof globalThis.window !== "undefined";
|
|
40
|
+
if (isBrowser) {
|
|
41
|
+
try {
|
|
42
|
+
// Dynamic import from CDN URL
|
|
43
|
+
const ort = (await import(
|
|
44
|
+
/* webpackIgnore: true */ ONNX_WEB_CDN_URL));
|
|
45
|
+
return ort;
|
|
46
|
+
}
|
|
47
|
+
catch (cdnError) {
|
|
48
|
+
throw new Error(`Failed to load onnxruntime-web from CDN: ${String(cdnError)}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
throw new Error("onnxruntime-web is not available");
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Loads the ONNX runtime (always onnxruntime-web in browser build)
|
|
56
|
+
*/
|
|
57
|
+
export async function loadRuntime(preferredRuntime) {
|
|
58
|
+
if (_runtime !== null) {
|
|
59
|
+
return _runtime;
|
|
60
|
+
}
|
|
61
|
+
// In browser build, we only support "web" runtime
|
|
62
|
+
if (preferredRuntime === "node") {
|
|
63
|
+
throw new Error("onnxruntime-node is not available in browser build.\n" +
|
|
64
|
+
"Import from 'rehydra' (not 'rehydra/browser') for Node.js support.");
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
const ort = await loadOnnxWeb();
|
|
68
|
+
_runtime = ort;
|
|
69
|
+
_runtimeType = "web";
|
|
70
|
+
}
|
|
71
|
+
catch (e) {
|
|
72
|
+
throw new Error(`Failed to load ONNX runtime. Make sure 'onnxruntime-web' is installed.\n` +
|
|
73
|
+
`Original error: ${String(e)}`);
|
|
74
|
+
}
|
|
75
|
+
return _runtime;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Gets the currently loaded runtime type
|
|
79
|
+
*/
|
|
80
|
+
export function getRuntimeType() {
|
|
81
|
+
return _runtimeType;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Resets the runtime (useful for testing)
|
|
85
|
+
*/
|
|
86
|
+
export function resetRuntime() {
|
|
87
|
+
_runtime = null;
|
|
88
|
+
_runtimeType = null;
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=onnx-runtime.browser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx-runtime.browser.js","sourceRoot":"","sources":["../../src/ner/onnx-runtime.browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,uEAAuE;AACvE,kEAAkE;AAClE,MAAM,gBAAgB,GACpB,6EAA6E,CAAC;AAkChF;;GAEG;AACH,IAAI,QAAQ,GAAsB,IAAI,CAAC;AACvC,IAAI,YAAY,GAA0B,IAAI,CAAC;AAE/C;;;GAGG;AACH,MAAM,UAAU,aAAa;IAC3B,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,6DAA6D;IAC7D,IAAI,CAAC;QACH,6DAA6D;QAC7D,oDAAoD;QACpD,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAe,CAAC;QAC5D,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,uEAAuE;QACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;QAC3D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,8BAA8B;gBAC9B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM;gBACvB,yBAAyB,CAAC,gBAAgB,CAC3C,CAAe,CAAC;gBACjB,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAC/D,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,gBAAiC;IAEjC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,kDAAkD;IAClD,IAAI,gBAAgB,KAAK,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,uDAAuD;YACvD,oEAAoE,CACrE,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;QAChC,QAAQ,GAAG,GAAG,CAAC;QACf,YAAY,GAAG,KAAK,CAAC;IACvB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,mBAAmB,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,QAAQ,GAAG,IAAI,CAAC;IAChB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ONNX Runtime Abstraction
|
|
3
|
+
* Allows switching between onnxruntime-node and onnxruntime-web
|
|
4
|
+
*
|
|
5
|
+
* In browsers without a bundler, automatically loads onnxruntime-web from CDN
|
|
6
|
+
*/
|
|
7
|
+
export interface OrtTensor {
|
|
8
|
+
data: Float32Array | BigInt64Array | Int32Array;
|
|
9
|
+
dims: readonly number[];
|
|
10
|
+
}
|
|
11
|
+
export interface OrtSession {
|
|
12
|
+
inputNames: readonly string[];
|
|
13
|
+
outputNames: readonly string[];
|
|
14
|
+
run(feeds: Record<string, OrtTensor>): Promise<Record<string, OrtTensor>>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* ONNX Runtime session options for performance tuning
|
|
18
|
+
*/
|
|
19
|
+
export interface OrtSessionOptions {
|
|
20
|
+
/**
|
|
21
|
+
* Execution providers in priority order.
|
|
22
|
+
* - For onnxruntime-web: ['webgpu', 'wasm'] or ['wasm']
|
|
23
|
+
* - For onnxruntime-node: usually not needed (uses CPU by default)
|
|
24
|
+
* - With custom builds: ['coreml'], etc.
|
|
25
|
+
*/
|
|
26
|
+
executionProviders?: Array<string | {
|
|
27
|
+
name: string;
|
|
28
|
+
[key: string]: unknown;
|
|
29
|
+
}>;
|
|
30
|
+
/** Graph optimization level */
|
|
31
|
+
graphOptimizationLevel?: "disabled" | "basic" | "extended" | "all";
|
|
32
|
+
/** Number of threads for parallel execution within operators */
|
|
33
|
+
intraOpNumThreads?: number;
|
|
34
|
+
/** Number of threads for parallel execution between operators */
|
|
35
|
+
interOpNumThreads?: number;
|
|
36
|
+
/** Enable CPU memory arena for allocations */
|
|
37
|
+
enableCpuMemArena?: boolean;
|
|
38
|
+
/** Enable memory pattern optimization */
|
|
39
|
+
enableMemPattern?: boolean;
|
|
40
|
+
}
|
|
41
|
+
export interface OrtInferenceSession {
|
|
42
|
+
create(pathOrBuffer: string | ArrayBuffer | Uint8Array, options?: OrtSessionOptions): Promise<OrtSession>;
|
|
43
|
+
}
|
|
44
|
+
export interface OrtTensorConstructor {
|
|
45
|
+
new (type: string, data: Float32Array | BigInt64Array | Int32Array | number[] | bigint[], dims: number[]): OrtTensor;
|
|
46
|
+
}
|
|
47
|
+
export interface OrtRuntime {
|
|
48
|
+
InferenceSession: OrtInferenceSession;
|
|
49
|
+
Tensor: OrtTensorConstructor;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Detects the best ONNX runtime for the current environment
|
|
53
|
+
*/
|
|
54
|
+
export declare function detectRuntime(): "node" | "web";
|
|
55
|
+
/**
|
|
56
|
+
* Loads the appropriate ONNX runtime
|
|
57
|
+
* @param preferredRuntime - Force 'node' or 'web' runtime
|
|
58
|
+
*/
|
|
59
|
+
export declare function loadRuntime(preferredRuntime?: "node" | "web"): Promise<OrtRuntime>;
|
|
60
|
+
/**
|
|
61
|
+
* Gets the currently loaded runtime type
|
|
62
|
+
*/
|
|
63
|
+
export declare function getRuntimeType(): "node" | "web" | null;
|
|
64
|
+
/**
|
|
65
|
+
* Resets the runtime (useful for testing)
|
|
66
|
+
*/
|
|
67
|
+
export declare function resetRuntime(): void;
|
|
68
|
+
declare global {
|
|
69
|
+
var Bun: unknown;
|
|
70
|
+
var Deno: unknown;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=onnx-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx-runtime.d.ts","sourceRoot":"","sources":["../../src/ner/onnx-runtime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,UAAU,CAAC;IAChD,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;CAC3E;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAC;IAC9E,+BAA+B;IAC/B,sBAAsB,CAAC,EAAE,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,KAAK,CAAC;IACnE,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8CAA8C;IAC9C,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,CACJ,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,UAAU,EAC/C,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,KACE,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,UAAU,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,EACrE,IAAI,EAAE,MAAM,EAAE,GACb,SAAS,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,gBAAgB,EAAE,mBAAmB,CAAC;IACtC,MAAM,EAAE,oBAAoB,CAAC;CAC9B;AAQD;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,GAAG,KAAK,CA4B9C;AAiCD;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,gBAAgB,CAAC,EAAE,MAAM,GAAG,KAAK,GAChC,OAAO,CAAC,UAAU,CAAC,CA4CrB;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,GAAG,KAAK,GAAG,IAAI,CAEtD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAGnC;AAGD,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,GAAG,EAAE,OAAO,CAAC;IAEjB,IAAI,IAAI,EAAE,OAAO,CAAC;CACnB"}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ONNX Runtime Abstraction
|
|
3
|
+
* Allows switching between onnxruntime-node and onnxruntime-web
|
|
4
|
+
*
|
|
5
|
+
* In browsers without a bundler, automatically loads onnxruntime-web from CDN
|
|
6
|
+
*/
|
|
7
|
+
// CDN URL for onnxruntime-web (used when bare import fails in browser)
|
|
8
|
+
// Using the bundled ESM version that includes WebAssembly backend
|
|
9
|
+
const ONNX_WEB_CDN_URL = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/ort.bundle.min.mjs";
|
|
10
|
+
/**
|
|
11
|
+
* Runtime detection and loading
|
|
12
|
+
*/
|
|
13
|
+
let _runtime = null;
|
|
14
|
+
let _runtimeType = null;
|
|
15
|
+
/**
|
|
16
|
+
* Detects the best ONNX runtime for the current environment
|
|
17
|
+
*/
|
|
18
|
+
export function detectRuntime() {
|
|
19
|
+
// Check if we're in Bun
|
|
20
|
+
const isBun = typeof globalThis.Bun !== "undefined";
|
|
21
|
+
// Check if we're in a browser-like environment
|
|
22
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
23
|
+
const isBrowser = typeof globalThis.window !== "undefined";
|
|
24
|
+
// Check if we're in Deno
|
|
25
|
+
const isDeno = typeof globalThis.Deno !== "undefined";
|
|
26
|
+
if (isBrowser || isDeno) {
|
|
27
|
+
return "web";
|
|
28
|
+
}
|
|
29
|
+
// For Bun, try node first, fall back to web
|
|
30
|
+
if (isBun) {
|
|
31
|
+
try {
|
|
32
|
+
// Quick check if onnxruntime-node is loadable
|
|
33
|
+
require.resolve("onnxruntime-node");
|
|
34
|
+
return "node";
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return "web";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
// Default to node for Node.js
|
|
41
|
+
return "node";
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Attempts to load onnxruntime-web, first via bare import, then via CDN
|
|
45
|
+
*/
|
|
46
|
+
async function loadOnnxWeb() {
|
|
47
|
+
// First try bare import (works with bundlers or import maps)
|
|
48
|
+
try {
|
|
49
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
50
|
+
// @ts-ignore - onnxruntime-web may not be installed
|
|
51
|
+
const ort = (await import("onnxruntime-web"));
|
|
52
|
+
return ort;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// Bare import failed, try CDN (for browsers without bundlers)
|
|
56
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
57
|
+
const isBrowser = typeof globalThis.window !== "undefined";
|
|
58
|
+
if (isBrowser) {
|
|
59
|
+
try {
|
|
60
|
+
// Dynamic import from CDN URL
|
|
61
|
+
const ort = (await import(
|
|
62
|
+
/* webpackIgnore: true */ ONNX_WEB_CDN_URL));
|
|
63
|
+
return ort;
|
|
64
|
+
}
|
|
65
|
+
catch (cdnError) {
|
|
66
|
+
throw new Error(`Failed to load onnxruntime-web from CDN: ${String(cdnError)}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
throw new Error("onnxruntime-web is not available");
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Loads the appropriate ONNX runtime
|
|
74
|
+
* @param preferredRuntime - Force 'node' or 'web' runtime
|
|
75
|
+
*/
|
|
76
|
+
export async function loadRuntime(preferredRuntime) {
|
|
77
|
+
// If runtime is already loaded, return it
|
|
78
|
+
if (_runtime !== null) {
|
|
79
|
+
return _runtime;
|
|
80
|
+
}
|
|
81
|
+
const runtimeType = preferredRuntime ?? detectRuntime();
|
|
82
|
+
try {
|
|
83
|
+
if (runtimeType === "node") {
|
|
84
|
+
// Load standard CPU runtime (works with Bun)
|
|
85
|
+
const ort = (await import("onnxruntime-node"));
|
|
86
|
+
_runtime = ort;
|
|
87
|
+
_runtimeType = "node";
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
// Load onnxruntime-web (with CDN fallback for browsers)
|
|
91
|
+
const ort = await loadOnnxWeb();
|
|
92
|
+
_runtime = ort;
|
|
93
|
+
_runtimeType = "web";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
catch (e) {
|
|
97
|
+
// If preferred runtime fails, try the other
|
|
98
|
+
const fallbackType = runtimeType === "node" ? "web" : "node";
|
|
99
|
+
try {
|
|
100
|
+
if (fallbackType === "node") {
|
|
101
|
+
const ort = (await import("onnxruntime-node"));
|
|
102
|
+
_runtime = ort;
|
|
103
|
+
_runtimeType = "node";
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
// Load onnxruntime-web (with CDN fallback for browsers)
|
|
107
|
+
const ort = await loadOnnxWeb();
|
|
108
|
+
_runtime = ort;
|
|
109
|
+
_runtimeType = "web";
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
throw new Error(`Failed to load ONNX runtime. Install either 'onnxruntime-node' or 'onnxruntime-web'.\n` +
|
|
114
|
+
`Original error: ${String(e)}`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return _runtime;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Gets the currently loaded runtime type
|
|
121
|
+
*/
|
|
122
|
+
export function getRuntimeType() {
|
|
123
|
+
return _runtimeType;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Resets the runtime (useful for testing)
|
|
127
|
+
*/
|
|
128
|
+
export function resetRuntime() {
|
|
129
|
+
_runtime = null;
|
|
130
|
+
_runtimeType = null;
|
|
131
|
+
}
|
|
132
|
+
//# sourceMappingURL=onnx-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx-runtime.js","sourceRoot":"","sources":["../../src/ner/onnx-runtime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,uEAAuE;AACvE,kEAAkE;AAClE,MAAM,gBAAgB,GACpB,6EAA6E,CAAC;AAyDhF;;GAEG;AACH,IAAI,QAAQ,GAAsB,IAAI,CAAC;AACvC,IAAI,YAAY,GAA0B,IAAI,CAAC;AAE/C;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,wBAAwB;IACxB,MAAM,KAAK,GAAG,OAAO,UAAU,CAAC,GAAG,KAAK,WAAW,CAAC;IAEpD,+CAA+C;IAC/C,uEAAuE;IACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;IAE3D,yBAAyB;IACzB,MAAM,MAAM,GAAG,OAAO,UAAU,CAAC,IAAI,KAAK,WAAW,CAAC;IAEtD,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,4CAA4C;IAC5C,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC;YACH,8CAA8C;YAC9C,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,6DAA6D;IAC7D,IAAI,CAAC;QACH,6DAA6D;QAC7D,oDAAoD;QACpD,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAe,CAAC;QAC5D,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,uEAAuE;QACvE,MAAM,SAAS,GAAG,OAAO,UAAU,CAAC,MAAM,KAAK,WAAW,CAAC;QAC3D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,8BAA8B;gBAC9B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM;gBACvB,yBAAyB,CAAC,gBAAgB,CAC3C,CAAe,CAAC;gBACjB,OAAO,GAAG,CAAC;YACb,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAC/D,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,gBAAiC;IAEjC,0CAA0C;IAC1C,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,gBAAgB,IAAI,aAAa,EAAE,CAAC;IAExD,IAAI,CAAC;QACH,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,6CAA6C;YAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAe,CAAC;YAC7D,QAAQ,GAAG,GAAG,CAAC;YACf,YAAY,GAAG,MAAM,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;YAChC,QAAQ,GAAG,GAAG,CAAC;YACf,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,4CAA4C;QAC5C,MAAM,YAAY,GAAG,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QAE7D,IAAI,CAAC;YACH,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAe,CAAC;gBAC7D,QAAQ,GAAG,GAAG,CAAC;gBACf,YAAY,GAAG,MAAM,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,wDAAwD;gBACxD,MAAM,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;gBAChC,QAAQ,GAAG,GAAG,CAAC;gBACf,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,wFAAwF;gBACtF,mBAAmB,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,QAAQ,GAAG,IAAI,CAAC;IAChB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { SpanMatch } from "../types/index.js";
|
|
2
|
+
import type { TokenizationResult } from "./tokenizer.js";
|
|
3
|
+
/**
|
|
4
|
+
* Total token count (including the two special tokens) that `maxWindows`
|
|
5
|
+
* windows can consume. Used to bound tokenization up front so huge inputs
|
|
6
|
+
* are not fully tokenized just to discard everything past the cap.
|
|
7
|
+
*/
|
|
8
|
+
export declare function windowTokenBudget(maxLength: number, overlapTokens: number, maxWindows: number): number;
|
|
9
|
+
export declare function validateWindowConfig(maxLength: number, overlapTokens: number, maxWindows?: number): void;
|
|
10
|
+
/**
|
|
11
|
+
* Splits an already-tokenized input into overlapping model-sized windows.
|
|
12
|
+
* The window count is bounded by the caller pre-bounding tokenization via
|
|
13
|
+
* `tokenize`'s `maxTokens` (see `runNERPass`) — this just slices whatever it
|
|
14
|
+
* is handed, so a truncated tokenization yields a truncated set of windows,
|
|
15
|
+
* and the truncation signal lives on the parent `TokenizationResult`.
|
|
16
|
+
*/
|
|
17
|
+
export declare function createTokenWindows(tokenization: TokenizationResult, maxLength: number, overlapTokens: number): TokenizationResult[];
|
|
18
|
+
export declare function mergeWindowSpans(spans: SpanMatch[], originalText: string): SpanMatch[];
|
|
19
|
+
//# sourceMappingURL=token-window.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-window.d.ts","sourceRoot":"","sources":["../../src/ner/token-window.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAS,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAiBhE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,GACjB,MAAM,CAKR;AAED,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,CAAC,EAAE,MAAM,GAClB,IAAI,CAUN;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,YAAY,EAAE,kBAAkB,EAChC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,GACpB,kBAAkB,EAAE,CA+BtB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAwCb"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
function buildTokenization(tokens) {
|
|
2
|
+
return {
|
|
3
|
+
tokens,
|
|
4
|
+
inputIds: tokens.map((token) => token.id),
|
|
5
|
+
attentionMask: tokens.map(() => 1),
|
|
6
|
+
tokenTypeIds: tokens.map(() => 0),
|
|
7
|
+
tokenToCharSpan: tokens.map((token) => token.isSpecial ? null : [token.start, token.end]),
|
|
8
|
+
// A window is a complete slice of an already-tokenized input; the
|
|
9
|
+
// truncated signal lives on the parent tokenization, not each window.
|
|
10
|
+
truncated: false,
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Total token count (including the two special tokens) that `maxWindows`
|
|
15
|
+
* windows can consume. Used to bound tokenization up front so huge inputs
|
|
16
|
+
* are not fully tokenized just to discard everything past the cap.
|
|
17
|
+
*/
|
|
18
|
+
export function windowTokenBudget(maxLength, overlapTokens, maxWindows) {
|
|
19
|
+
validateWindowConfig(maxLength, overlapTokens, maxWindows);
|
|
20
|
+
const contentCapacity = maxLength - 2;
|
|
21
|
+
const stride = contentCapacity - overlapTokens;
|
|
22
|
+
return 2 + contentCapacity + (maxWindows - 1) * stride;
|
|
23
|
+
}
|
|
24
|
+
export function validateWindowConfig(maxLength, overlapTokens, maxWindows) {
|
|
25
|
+
if (maxLength - 2 < 1) {
|
|
26
|
+
throw new Error("NER maxLength must leave room for content and two special tokens");
|
|
27
|
+
}
|
|
28
|
+
if (overlapTokens < 0 || overlapTokens >= maxLength - 2) {
|
|
29
|
+
throw new Error("NER window overlap must be smaller than the content capacity");
|
|
30
|
+
}
|
|
31
|
+
if (maxWindows !== undefined && maxWindows < 1) {
|
|
32
|
+
throw new Error("NER maxWindowsPerInput must be at least 1");
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Splits an already-tokenized input into overlapping model-sized windows.
|
|
37
|
+
* The window count is bounded by the caller pre-bounding tokenization via
|
|
38
|
+
* `tokenize`'s `maxTokens` (see `runNERPass`) — this just slices whatever it
|
|
39
|
+
* is handed, so a truncated tokenization yields a truncated set of windows,
|
|
40
|
+
* and the truncation signal lives on the parent `TokenizationResult`.
|
|
41
|
+
*/
|
|
42
|
+
export function createTokenWindows(tokenization, maxLength, overlapTokens) {
|
|
43
|
+
validateWindowConfig(maxLength, overlapTokens);
|
|
44
|
+
const contentCapacity = maxLength - 2;
|
|
45
|
+
if (tokenization.tokens.length <= maxLength) {
|
|
46
|
+
return [tokenization];
|
|
47
|
+
}
|
|
48
|
+
const firstToken = tokenization.tokens[0];
|
|
49
|
+
const lastToken = tokenization.tokens[tokenization.tokens.length - 1];
|
|
50
|
+
if (firstToken === undefined ||
|
|
51
|
+
firstToken.isSpecial !== true ||
|
|
52
|
+
lastToken === undefined ||
|
|
53
|
+
lastToken.isSpecial !== true) {
|
|
54
|
+
throw new Error("NER tokenization must include leading and trailing special tokens");
|
|
55
|
+
}
|
|
56
|
+
const contentTokens = tokenization.tokens.slice(1, -1);
|
|
57
|
+
const stride = contentCapacity - overlapTokens;
|
|
58
|
+
const windows = [];
|
|
59
|
+
for (let start = 0; start < contentTokens.length; start += stride) {
|
|
60
|
+
const end = Math.min(start + contentCapacity, contentTokens.length);
|
|
61
|
+
windows.push(buildTokenization([firstToken, ...contentTokens.slice(start, end), lastToken]));
|
|
62
|
+
if (end === contentTokens.length)
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
return windows;
|
|
66
|
+
}
|
|
67
|
+
export function mergeWindowSpans(spans, originalText) {
|
|
68
|
+
const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - a.end || b.confidence - a.confidence);
|
|
69
|
+
const merged = [];
|
|
70
|
+
for (const span of sorted) {
|
|
71
|
+
let combined = span;
|
|
72
|
+
for (let index = merged.length - 1; index >= 0; index--) {
|
|
73
|
+
const existing = merged[index];
|
|
74
|
+
// Only stitch spans that actually overlap (touching end-to-start is not
|
|
75
|
+
// an overlap). This relies on the window overlap being at least as long
|
|
76
|
+
// as the longest expected entity — otherwise an entity split across a
|
|
77
|
+
// window boundary produces two abutting fragments that never merge.
|
|
78
|
+
// The default overlap (64 tokens) is far longer than any name/entity.
|
|
79
|
+
if (existing.type !== combined.type ||
|
|
80
|
+
existing.start >= combined.end ||
|
|
81
|
+
combined.start >= existing.end) {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
const preferred = combined.confidence > existing.confidence
|
|
85
|
+
? combined
|
|
86
|
+
: existing;
|
|
87
|
+
const start = Math.min(existing.start, combined.start);
|
|
88
|
+
const end = Math.max(existing.end, combined.end);
|
|
89
|
+
combined = {
|
|
90
|
+
...preferred,
|
|
91
|
+
start,
|
|
92
|
+
end,
|
|
93
|
+
text: originalText.slice(start, end),
|
|
94
|
+
};
|
|
95
|
+
merged.splice(index, 1);
|
|
96
|
+
}
|
|
97
|
+
merged.push(combined);
|
|
98
|
+
}
|
|
99
|
+
return merged.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=token-window.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-window.js","sourceRoot":"","sources":["../../src/ner/token-window.ts"],"names":[],"mappings":"AAGA,SAAS,iBAAiB,CAAC,MAAe;IACxC,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACjC,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACpC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAClD;QACD,kEAAkE;QAClE,sEAAsE;QACtE,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAAiB,EACjB,aAAqB,EACrB,UAAkB;IAElB,oBAAoB,CAAC,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,eAAe,GAAG,aAAa,CAAC;IAC/C,OAAO,CAAC,GAAG,eAAe,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,SAAiB,EACjB,aAAqB,EACrB,UAAmB;IAEnB,IAAI,SAAS,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,aAAa,GAAG,CAAC,IAAI,aAAa,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,YAAgC,EAChC,SAAiB,EACjB,aAAqB;IAErB,oBAAoB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;IACtC,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC5C,OAAO,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtE,IACE,UAAU,KAAK,SAAS;QACxB,UAAU,CAAC,SAAS,KAAK,IAAI;QAC7B,SAAS,KAAK,SAAS;QACvB,SAAS,CAAC,SAAS,KAAK,IAAI,EAC5B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,eAAe,GAAG,aAAa,CAAC;IAC/C,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,eAAe,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CACV,iBAAiB,CAAC,CAAC,UAAU,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAC,CAC/E,CAAC;QACF,IAAI,GAAG,KAAK,aAAa,CAAC,MAAM;YAAE,MAAM;IAC1C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,KAAkB,EAClB,YAAoB;IAEpB,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAClE,CAAC;IACF,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,KAAK,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;YAChC,wEAAwE;YACxE,wEAAwE;YACxE,sEAAsE;YACtE,oEAAoE;YACpE,sEAAsE;YACtE,IACE,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;gBAC/B,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG;gBAC9B,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,EAC9B,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU;gBACzD,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACjD,QAAQ,GAAG;gBACT,GAAG,SAAS;gBACZ,KAAK;gBACL,GAAG;gBACH,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;aACrC,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HuggingFace Tokenizer
|
|
3
|
+
* Loads and uses tokenizers from HuggingFace's tokenizer.json format
|
|
4
|
+
* Supports Unigram (SentencePiece) and BPE tokenizers
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Token with offset information
|
|
8
|
+
*/
|
|
9
|
+
export interface Token {
|
|
10
|
+
/** Token ID in vocabulary */
|
|
11
|
+
id: number;
|
|
12
|
+
/** Token string */
|
|
13
|
+
token: string;
|
|
14
|
+
/** Start character offset in original text */
|
|
15
|
+
start: number;
|
|
16
|
+
/** End character offset in original text */
|
|
17
|
+
end: number;
|
|
18
|
+
/** Whether this is a continuation token */
|
|
19
|
+
isContinuation: boolean;
|
|
20
|
+
/** Whether this is a special token */
|
|
21
|
+
isSpecial: boolean;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Tokenization result with metadata
|
|
25
|
+
*/
|
|
26
|
+
export interface TokenizationResult {
|
|
27
|
+
/** Array of tokens */
|
|
28
|
+
tokens: Token[];
|
|
29
|
+
/** Input IDs for model */
|
|
30
|
+
inputIds: number[];
|
|
31
|
+
/** Attention mask */
|
|
32
|
+
attentionMask: number[];
|
|
33
|
+
/** Token type IDs (for BERT-style models) */
|
|
34
|
+
tokenTypeIds: number[];
|
|
35
|
+
/** Mapping from token index to character span [start, end] */
|
|
36
|
+
tokenToCharSpan: Array<[number, number] | null>;
|
|
37
|
+
/**
|
|
38
|
+
* True when tokenization stopped before consuming all input — either
|
|
39
|
+
* because `truncate` capped it at maxLength or because `maxTokens` bounded
|
|
40
|
+
* it. Lets callers know detection coverage is partial.
|
|
41
|
+
*/
|
|
42
|
+
truncated: boolean;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Tokenizer configuration
|
|
46
|
+
*/
|
|
47
|
+
export interface TokenizerConfig {
|
|
48
|
+
/** Maximum sequence length */
|
|
49
|
+
maxLength: number;
|
|
50
|
+
/** Whether to lowercase input */
|
|
51
|
+
doLowerCase: boolean;
|
|
52
|
+
}
|
|
53
|
+
export interface TokenizeOptions {
|
|
54
|
+
/** Whether to truncate to the configured maximum sequence length. */
|
|
55
|
+
truncate?: boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Stop after producing this many tokens (including the CLS/SEP special
|
|
58
|
+
* tokens). Bounds memory and CPU on very large inputs: the caller only
|
|
59
|
+
* needs enough tokens to build its windows, so there is no point
|
|
60
|
+
* materializing a multi-million-token array for a multi-MB value.
|
|
61
|
+
* Takes precedence over `truncate` when set.
|
|
62
|
+
*/
|
|
63
|
+
maxTokens?: number;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Default tokenizer configuration
|
|
67
|
+
*/
|
|
68
|
+
export declare const DEFAULT_TOKENIZER_CONFIG: TokenizerConfig;
|
|
69
|
+
/**
|
|
70
|
+
* WordPiece Tokenizer - supports both HuggingFace JSON and vocab.txt formats
|
|
71
|
+
* Uses a Trie for O(token_length) lookups instead of O(vocab_size)
|
|
72
|
+
*/
|
|
73
|
+
export declare class WordPieceTokenizer {
|
|
74
|
+
private vocab;
|
|
75
|
+
private inverseVocab;
|
|
76
|
+
private config;
|
|
77
|
+
private trie;
|
|
78
|
+
private clsId;
|
|
79
|
+
private sepId;
|
|
80
|
+
private padId;
|
|
81
|
+
private unkId;
|
|
82
|
+
private clsToken;
|
|
83
|
+
private sepToken;
|
|
84
|
+
private padToken;
|
|
85
|
+
private unkToken;
|
|
86
|
+
constructor(vocab: Map<string, number>, config?: Partial<TokenizerConfig>);
|
|
87
|
+
/**
|
|
88
|
+
* Insert a token into the trie
|
|
89
|
+
*/
|
|
90
|
+
private insertIntoTrie;
|
|
91
|
+
/**
|
|
92
|
+
* Detect special tokens from vocabulary
|
|
93
|
+
*/
|
|
94
|
+
private detectSpecialTokens;
|
|
95
|
+
/**
|
|
96
|
+
* Tokenizes text into tokens with offset tracking
|
|
97
|
+
*/
|
|
98
|
+
tokenize(text: string, options?: TokenizeOptions): TokenizationResult;
|
|
99
|
+
/**
|
|
100
|
+
* Find the best matching token using trie-based greedy longest-match
|
|
101
|
+
* O(max_token_length) instead of O(vocab_size)
|
|
102
|
+
*/
|
|
103
|
+
private findBestToken;
|
|
104
|
+
/**
|
|
105
|
+
* Find the longest matching token in the trie
|
|
106
|
+
*/
|
|
107
|
+
private findLongestTrieMatch;
|
|
108
|
+
/**
|
|
109
|
+
* Decodes token IDs back to text
|
|
110
|
+
*/
|
|
111
|
+
decode(tokenIds: number[]): string;
|
|
112
|
+
/**
|
|
113
|
+
* Gets vocabulary size
|
|
114
|
+
*/
|
|
115
|
+
get vocabSize(): number;
|
|
116
|
+
/**
|
|
117
|
+
* Gets a token ID by string
|
|
118
|
+
*/
|
|
119
|
+
getTokenId(token: string): number | undefined;
|
|
120
|
+
/**
|
|
121
|
+
* Gets a token string by ID
|
|
122
|
+
*/
|
|
123
|
+
getToken(id: number): string | undefined;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Loads vocabulary from a file (supports tokenizer.json and vocab.txt)
|
|
127
|
+
* Uses storage abstraction for browser compatibility
|
|
128
|
+
*/
|
|
129
|
+
export declare function loadVocabFromFile(filePath: string): Promise<Map<string, number>>;
|
|
130
|
+
/**
|
|
131
|
+
* Parses HuggingFace tokenizer.json format
|
|
132
|
+
*/
|
|
133
|
+
export declare function parseHFTokenizerJson(content: string): Map<string, number>;
|
|
134
|
+
/**
|
|
135
|
+
* Parses vocabulary from string content (vocab.txt format)
|
|
136
|
+
*/
|
|
137
|
+
export declare function parseVocab(content: string): Map<string, number>;
|
|
138
|
+
/**
|
|
139
|
+
* Creates a minimal vocabulary for testing
|
|
140
|
+
*/
|
|
141
|
+
export declare function createTestVocab(): Map<string, number>;
|
|
142
|
+
//# sourceMappingURL=tokenizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,mBAAmB;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,cAAc,EAAE,OAAO,CAAC;IACxB,sCAAsC;IACtC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,sBAAsB;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8DAA8D;IAC9D,eAAe,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAChD;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;CACpB;AAsBD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAkBF;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,IAAI,CAAW;IAGvB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAsB7E;;OAEG;IACH,OAAO,CAAC,cAAc;IAYtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,kBAAkB;IAqFzE;;;OAGG;IACH,OAAO,CAAC,aAAa;IAyBrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsC5B;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAmBlC;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;CAGzC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAWtF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAmCzE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAY/D;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrD"}
|