@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -4,6 +4,7 @@ import { transcribe, transcribeStream } from '../../../plugins/ops/transcribe.js
|
|
|
4
4
|
import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
5
5
|
import { attachBackendDiagnostics } from '../../../profiling/backend-diagnostics.js';
|
|
6
6
|
import { buildWhisperEngineConfig } from '../../../plugins/builtin/asr-ggml/config.js';
|
|
7
|
+
import { whisperFitRequest } from '../../../plugins/builtin/asr-ggml/fit.js';
|
|
7
8
|
import { createAsrModelLogger } from '../../../plugins/builtin/asr-ggml/logging.js';
|
|
8
9
|
function createWhisperModel(modelId, modelPath, whisperConfig, vadModelPath) {
|
|
9
10
|
const logger = createAsrModelLogger(modelId, ModelType.whispercppTranscription);
|
|
@@ -23,6 +24,7 @@ export const whisperPlugin = definePlugin({
|
|
|
23
24
|
displayName: 'Whisper (whisper.cpp)',
|
|
24
25
|
addonPackage: ADDON_ASR,
|
|
25
26
|
loadConfigSchema: whisperConfigSchema,
|
|
27
|
+
fitRequest: whisperFitRequest,
|
|
26
28
|
assessFit: ASRGgml.assessFit,
|
|
27
29
|
async resolveConfig(cfg, ctx) {
|
|
28
30
|
const { vadModelSrc, ...whisperConfig } = cfg;
|
|
@@ -48,7 +50,7 @@ export const whisperPlugin = definePlugin({
|
|
|
48
50
|
// whisper.cpp addon exposes a model-wide hard cancel — compute
|
|
49
51
|
// is interrupted on the currently-running transcription.
|
|
50
52
|
cancel: { scope: 'model', hard: true },
|
|
51
|
-
handler: async function* (request) {
|
|
53
|
+
handler: async function* (request, _inputStream, context) {
|
|
52
54
|
const metadata = request.metadata === true;
|
|
53
55
|
const stream = metadata
|
|
54
56
|
? transcribe({
|
|
@@ -58,14 +60,14 @@ export const whisperPlugin = definePlugin({
|
|
|
58
60
|
hotwords: request.hotwords,
|
|
59
61
|
maxNewTokens: request.maxNewTokens,
|
|
60
62
|
metadata: true
|
|
61
|
-
}, request.requestId)
|
|
63
|
+
}, request.requestId, request.sessionId, context)
|
|
62
64
|
: transcribe({
|
|
63
65
|
modelId: request.modelId,
|
|
64
66
|
audioChunk: request.audioChunk,
|
|
65
67
|
prompt: request.prompt,
|
|
66
68
|
hotwords: request.hotwords,
|
|
67
69
|
maxNewTokens: request.maxNewTokens
|
|
68
|
-
}, request.requestId);
|
|
70
|
+
}, request.requestId, request.sessionId, context);
|
|
69
71
|
try {
|
|
70
72
|
let result = await stream.next();
|
|
71
73
|
while (!result.done) {
|
|
@@ -105,7 +107,7 @@ export const whisperPlugin = definePlugin({
|
|
|
105
107
|
// Same model-wide hard cancel surface as `transcribe` — both
|
|
106
108
|
// route through the whisper.cpp addon.
|
|
107
109
|
cancel: { scope: 'model', hard: true },
|
|
108
|
-
handler: async function* (request, inputStream) {
|
|
110
|
+
handler: async function* (request, inputStream, context) {
|
|
109
111
|
const streamOpts = {
|
|
110
112
|
...(request.emitVadEvents !== undefined && {
|
|
111
113
|
emitVadEvents: request.emitVadEvents
|
|
@@ -119,8 +121,8 @@ export const whisperPlugin = definePlugin({
|
|
|
119
121
|
};
|
|
120
122
|
const metadata = request.metadata === true;
|
|
121
123
|
const iterator = metadata
|
|
122
|
-
? transcribeStream(request.modelId, inputStream, request.prompt, true, streamOpts, request.requestId)
|
|
123
|
-
: transcribeStream(request.modelId, inputStream, request.prompt, false, streamOpts, request.requestId);
|
|
124
|
+
? transcribeStream(request.modelId, inputStream, request.prompt, true, streamOpts, request.requestId, request.sessionId, context)
|
|
125
|
+
: transcribeStream(request.modelId, inputStream, request.prompt, false, streamOpts, request.requestId, request.sessionId, context);
|
|
124
126
|
try {
|
|
125
127
|
let result = await iterator.next();
|
|
126
128
|
while (!result.done) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../schemas/plugin.js';
|
|
1
2
|
import { type BciStreamOpts, type BciTranscribeParams, type TranscribeSegment, type TranscribeStats } from '../../schemas/index.js';
|
|
2
3
|
import Buffer from 'bare-buffer';
|
|
3
4
|
import { type StreamResult } from '../../profiling/model-execution.js';
|
|
@@ -9,9 +10,9 @@ type BciTranscribeReturn = {
|
|
|
9
10
|
};
|
|
10
11
|
export declare function bciTranscribe(params: BciTranscribeParams & {
|
|
11
12
|
metadata: true;
|
|
12
|
-
}, requestId?: string): AsyncGenerator<TranscribeSegment, BciTranscribeReturn, void>;
|
|
13
|
-
export declare function bciTranscribe(params: BciTranscribeParams, requestId?: string): AsyncGenerator<string, BciTranscribeReturn, void>;
|
|
13
|
+
}, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, BciTranscribeReturn, void>;
|
|
14
|
+
export declare function bciTranscribe(params: BciTranscribeParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, BciTranscribeReturn, void>;
|
|
14
15
|
type BciStreamReturn = StreamResult;
|
|
15
|
-
export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata: true, opts?: BciStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment, BciStreamReturn, void>;
|
|
16
|
-
export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata?: boolean, opts?: BciStreamOpts, requestId?: string): AsyncGenerator<string, BciStreamReturn, void>;
|
|
16
|
+
export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata: true, opts?: BciStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, BciStreamReturn, void>;
|
|
17
|
+
export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata?: boolean, opts?: BciStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, BciStreamReturn, void>;
|
|
17
18
|
export {};
|
|
@@ -50,10 +50,11 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
54
|
import { getModel } from '../../runtime/model-registry.js';
|
|
54
55
|
import Buffer from 'bare-buffer';
|
|
55
56
|
import { getEngineLogger } from '../../logging/index.js';
|
|
56
|
-
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
57
|
+
import { InferenceCancelledError, TranscriptionFailedError } from '../../errors/index.js';
|
|
57
58
|
import { nowMs } from '../../profiling/index.js';
|
|
58
59
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
59
60
|
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
@@ -70,7 +71,7 @@ async function runBci(model, input) {
|
|
|
70
71
|
throw new TranscriptionFailedError('Invalid neural input');
|
|
71
72
|
}
|
|
72
73
|
}
|
|
73
|
-
export async function* bciTranscribe(params, requestId) {
|
|
74
|
+
export async function* bciTranscribe(params, requestId, sessionId, context) {
|
|
74
75
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
75
76
|
try {
|
|
76
77
|
const { modelId, metadata } = params;
|
|
@@ -80,9 +81,18 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
80
81
|
// caller didn't send one.
|
|
81
82
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
82
83
|
requestId: requestId ?? generateRandomRequestId(),
|
|
84
|
+
sessionId,
|
|
85
|
+
...requestParent(context),
|
|
83
86
|
kind: 'transcribe',
|
|
84
87
|
modelId
|
|
85
|
-
})
|
|
88
|
+
})
|
|
89
|
+
// Cancelled while queued for the model: it holds no slot, so wiring the
|
|
90
|
+
// model-wide cancel below would stop the job of whoever runs now.
|
|
91
|
+
, true);
|
|
92
|
+
// Cancelled while queued for the model: it holds no slot, so wiring the
|
|
93
|
+
// model-wide cancel below would stop the job of whoever runs now.
|
|
94
|
+
if (ctx.signal.aborted)
|
|
95
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
86
96
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
87
97
|
const model = getModel(modelId);
|
|
88
98
|
// Hard-cancel wiring: the BCI addon exposes a model-wide `cancel()`
|
|
@@ -181,7 +191,7 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
181
191
|
await result_1;
|
|
182
192
|
}
|
|
183
193
|
}
|
|
184
|
-
export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts, requestId) {
|
|
194
|
+
export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts, requestId, sessionId, context) {
|
|
185
195
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
186
196
|
try {
|
|
187
197
|
// Same `kind: "transcribe"` as the unary BCI variant — the registry
|
|
@@ -190,9 +200,13 @@ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts
|
|
|
190
200
|
// either shape.
|
|
191
201
|
const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
|
|
192
202
|
requestId: requestId ?? generateRandomRequestId(),
|
|
203
|
+
sessionId,
|
|
204
|
+
...requestParent(context),
|
|
193
205
|
kind: 'transcribe',
|
|
194
206
|
modelId
|
|
195
207
|
}), true);
|
|
208
|
+
if (ctx.signal.aborted)
|
|
209
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
196
210
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
197
211
|
const model = getModel(modelId);
|
|
198
212
|
// Hard-cancel wiring: `cancel()` tears down the active stream driver
|
|
@@ -2,6 +2,8 @@ import { getModel } from '../../runtime/model-registry.js';
|
|
|
2
2
|
import { cancelInferenceBaseSchema } from '../../schemas/index.js';
|
|
3
3
|
import { ModelNotLoadedError } from '../../errors/index.js';
|
|
4
4
|
import { getRequestRegistry } from '../../runtime/index.js';
|
|
5
|
+
import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
|
|
6
|
+
import { isOwnerSession } from '../../runtime/session-registry.js';
|
|
5
7
|
import { getEngineLogger } from '../../logging/index.js';
|
|
6
8
|
const logger = getEngineLogger();
|
|
7
9
|
/**
|
|
@@ -30,9 +32,16 @@ export function cancel(params, opts) {
|
|
|
30
32
|
if (!model) {
|
|
31
33
|
throw new ModelNotLoadedError(modelId);
|
|
32
34
|
}
|
|
35
|
+
// The owner sweeps every session's requests on the model; a session only
|
|
36
|
+
// reaches its own.
|
|
37
|
+
const sessionId = opts?.sessionId ?? OWNER_SESSION_ID;
|
|
33
38
|
const registry = getRequestRegistry();
|
|
34
|
-
const
|
|
35
|
-
|
|
39
|
+
const cancelled = registry.cancel({
|
|
40
|
+
modelId,
|
|
41
|
+
...(opts?.kind && { kind: opts.kind }),
|
|
42
|
+
sessionId,
|
|
43
|
+
allSessions: isOwnerSession(sessionId)
|
|
44
|
+
});
|
|
36
45
|
if (cancelled === 0) {
|
|
37
46
|
// Callers (workbench "Stop" button, app shutdown sweeps) often
|
|
38
47
|
// fire-and-forget; log so operators can see when a broad cancel
|
|
@@ -1,15 +1,17 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../schemas/plugin.js';
|
|
1
2
|
import { type EmbedParams, type EmbedStats } from '../../schemas/index.js';
|
|
2
3
|
export interface EmbedResult {
|
|
3
4
|
embedding: number[] | number[][];
|
|
4
5
|
stats?: EmbedStats;
|
|
5
6
|
}
|
|
7
|
+
export declare function markEmbeddingProjector(model: object): void;
|
|
6
8
|
export declare function embed(params: {
|
|
7
9
|
modelId: string;
|
|
8
10
|
text: string;
|
|
9
|
-
}, requestId?: string): Promise<EmbedResult>;
|
|
11
|
+
}, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<EmbedResult>;
|
|
10
12
|
export declare function embed(params: {
|
|
11
13
|
modelId: string;
|
|
12
14
|
text: string[];
|
|
13
|
-
}, requestId?: string): Promise<EmbedResult>;
|
|
14
|
-
export declare function embed(params: EmbedParams, requestId?: string): Promise<EmbedResult>;
|
|
15
|
+
}, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<EmbedResult>;
|
|
16
|
+
export declare function embed(params: EmbedParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<EmbedResult>;
|
|
15
17
|
export declare function normalizeVector(vector: Float32Array): number[];
|
|
@@ -50,6 +50,9 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import fs from 'bare-fs';
|
|
54
|
+
import Buffer from 'bare-buffer';
|
|
55
|
+
import { requestParent } from '../../schemas/plugin.js';
|
|
53
56
|
import { getModel } from '../../runtime/model-registry.js';
|
|
54
57
|
import { embedParamsSchema } from '../../schemas/index.js';
|
|
55
58
|
import { buildUnaryResult } from '../../profiling/model-execution.js';
|
|
@@ -58,25 +61,35 @@ import { nowMs } from '../../profiling/index.js';
|
|
|
58
61
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
59
62
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
60
63
|
import { getEngineLogger } from '../../logging/index.js';
|
|
61
|
-
|
|
64
|
+
const modelsWithProjector = new WeakSet();
|
|
65
|
+
export function markEmbeddingProjector(model) {
|
|
66
|
+
modelsWithProjector.add(model);
|
|
67
|
+
}
|
|
68
|
+
export async function embed(params, requestId, sessionId, context) {
|
|
62
69
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
63
70
|
try {
|
|
64
|
-
const
|
|
71
|
+
const parsed = embedParamsSchema.parse(params);
|
|
72
|
+
const inputs = parsed.input !== undefined ? [parsed.input] : parsed.inputs;
|
|
73
|
+
const mediaInputs = inputs?.map((input) => {
|
|
74
|
+
if (typeof input === 'object' && !Array.isArray(input)) {
|
|
75
|
+
throw new EmbedFailedError('This model accepts text and ordered media parts, not raw RGB image inputs');
|
|
76
|
+
}
|
|
77
|
+
return input;
|
|
78
|
+
});
|
|
79
|
+
const { modelId } = parsed;
|
|
65
80
|
// Open a request-scoped lifecycle. The registry routes
|
|
66
81
|
// `cancel({ requestId })` and broad `cancel({ modelId, kind: "embeddings" })`
|
|
67
82
|
// straight to this context's signal. Falls back to a generated
|
|
68
83
|
// id if the caller didn't send one.
|
|
69
84
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
70
85
|
requestId: requestId ?? generateRandomRequestId(),
|
|
86
|
+
sessionId,
|
|
87
|
+
...requestParent(context),
|
|
71
88
|
kind: 'embeddings',
|
|
72
89
|
modelId
|
|
73
|
-
})
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
// shape, even when this op has no per-step emits beyond the
|
|
77
|
-
// registry's own lifecycle lines. Future addon-level warns inside
|
|
78
|
-
// this body should route through `requestLogger`.
|
|
79
|
-
, true);
|
|
90
|
+
}), true);
|
|
91
|
+
if (ctx.signal.aborted)
|
|
92
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
80
93
|
// `requestLogger` is intentionally referenced once so the
|
|
81
94
|
// request-scoped logger is built at handler entry per the canonical
|
|
82
95
|
// shape, even when this op has no per-step emits beyond the
|
|
@@ -84,42 +97,47 @@ export async function embed(params, requestId) {
|
|
|
84
97
|
// this body should route through `requestLogger`.
|
|
85
98
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
86
99
|
const model = getModel(modelId);
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
100
|
+
const hasMedia = mediaInputs?.some((input) => Array.isArray(input) && input.some((part) => typeof part !== 'string'));
|
|
101
|
+
if (hasMedia && !modelsWithProjector.has(model)) {
|
|
102
|
+
throw new EmbedFailedError('Media inputs require modelConfig.projectionModelSrc when loading the embedding model');
|
|
103
|
+
}
|
|
104
|
+
const input = mediaInputs !== undefined
|
|
105
|
+
? await prepareEmbedInputs(mediaInputs, ctx.signal, ctx.requestId)
|
|
106
|
+
: parsed.text;
|
|
107
|
+
if (ctx.signal.aborted)
|
|
108
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
109
|
+
// Admission keeps model-wide cancellation scoped to this request. Media
|
|
110
|
+
// preparation has no native job to cancel, so wire the addon only afterward.
|
|
111
|
+
let cancelPromise;
|
|
96
112
|
const onAbort = () => {
|
|
97
113
|
const addon = model.addon;
|
|
98
114
|
if (addon?.cancel) {
|
|
99
|
-
addon.cancel.call(addon).catch((err) => {
|
|
115
|
+
cancelPromise ??= addon.cancel.call(addon).catch((err) => {
|
|
100
116
|
requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
101
117
|
});
|
|
102
118
|
}
|
|
103
119
|
};
|
|
104
120
|
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
105
|
-
|
|
106
|
-
// register time. The registry may abort synchronously when a
|
|
107
|
-
// cancel-before-begin race resolves or the parent signal was pre-
|
|
108
|
-
// aborted — re-use `onAbort` so the listener body is the single
|
|
109
|
-
// source of truth for "what cancel does."
|
|
110
|
-
if (ctx.signal.aborted)
|
|
111
|
-
onAbort();
|
|
112
|
-
ctx.scope.defer(() => {
|
|
121
|
+
ctx.scope.defer(async () => {
|
|
113
122
|
ctx.signal.removeEventListener('abort', onAbort);
|
|
123
|
+
// A delayed native cancel must finish before the next job owns this model slot.
|
|
124
|
+
await cancelPromise;
|
|
114
125
|
});
|
|
115
126
|
const modelStart = nowMs();
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
127
|
+
let response;
|
|
128
|
+
let rawEmbeddings;
|
|
129
|
+
try {
|
|
130
|
+
response = (await model.run(input));
|
|
131
|
+
if (ctx.signal.aborted)
|
|
132
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
133
|
+
rawEmbeddings = await response.await();
|
|
134
|
+
if (ctx.signal.aborted)
|
|
135
|
+
throw new InferenceCancelledError(ctx.requestId);
|
|
119
136
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
137
|
+
catch (cause) {
|
|
138
|
+
if (ctx.signal.aborted)
|
|
139
|
+
throw new InferenceCancelledError(ctx.requestId, {}, cause);
|
|
140
|
+
throw cause;
|
|
123
141
|
}
|
|
124
142
|
const modelExecutionMs = nowMs() - modelStart;
|
|
125
143
|
const stats = {
|
|
@@ -134,7 +152,7 @@ export async function embed(params, requestId) {
|
|
|
134
152
|
...(response.stats?.context_size !== undefined && { contextSize: response.stats.context_size })
|
|
135
153
|
};
|
|
136
154
|
const embeddingsArray = rawEmbeddings[0];
|
|
137
|
-
if (Array.isArray(text)) {
|
|
155
|
+
if (parsed.input === undefined && (parsed.inputs !== undefined || Array.isArray(parsed.text))) {
|
|
138
156
|
if (!embeddingsArray || embeddingsArray.length === 0) {
|
|
139
157
|
throw new EmbedNoEmbeddingsError();
|
|
140
158
|
}
|
|
@@ -191,3 +209,40 @@ export function normalizeVector(vector) {
|
|
|
191
209
|
}
|
|
192
210
|
return normalizedVector;
|
|
193
211
|
}
|
|
212
|
+
async function prepareEmbedInputs(inputs, signal, requestId) {
|
|
213
|
+
const result = [];
|
|
214
|
+
for (const input of inputs) {
|
|
215
|
+
if (signal.aborted)
|
|
216
|
+
throw new InferenceCancelledError(requestId);
|
|
217
|
+
if (typeof input === 'string') {
|
|
218
|
+
result.push(input);
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const parts = [];
|
|
222
|
+
for (const part of input) {
|
|
223
|
+
if (signal.aborted)
|
|
224
|
+
throw new InferenceCancelledError(requestId);
|
|
225
|
+
if (typeof part === 'string') {
|
|
226
|
+
parts.push(part);
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
const { type, value } = part.media;
|
|
230
|
+
try {
|
|
231
|
+
const media = type === 'base64'
|
|
232
|
+
? Buffer.from(value, 'base64')
|
|
233
|
+
: await fs.promises.readFile(value, 'buffer');
|
|
234
|
+
parts.push({ media });
|
|
235
|
+
}
|
|
236
|
+
catch (cause) {
|
|
237
|
+
if (signal.aborted)
|
|
238
|
+
throw new InferenceCancelledError(requestId, {}, cause);
|
|
239
|
+
throw new EmbedFailedError('Unable to read embedding media', cause);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
result.push(parts);
|
|
244
|
+
}
|
|
245
|
+
if (signal.aborted)
|
|
246
|
+
throw new InferenceCancelledError(requestId);
|
|
247
|
+
return { inputs: result };
|
|
248
|
+
}
|
|
@@ -3,6 +3,7 @@ import type { Tool } from '../../schemas/tools.js';
|
|
|
3
3
|
export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
|
|
4
4
|
export declare function generateConfigHash(systemPrompt: string | null): string;
|
|
5
5
|
export declare function generateCacheKey(messages: CacheMessage[], tools?: readonly Tool[]): string;
|
|
6
|
+
export declare function resolveCacheFilePath(modelId: string, configHash: string, cacheKey: string): string;
|
|
6
7
|
export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
|
|
7
8
|
export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
|
|
8
9
|
cacheKey: string;
|
|
@@ -12,7 +12,9 @@ const logger = getEngineLogger();
|
|
|
12
12
|
// touch in-memory state.
|
|
13
13
|
export function extractSystemPrompt(messages) {
|
|
14
14
|
const systemMessage = messages.find((msg) => msg.role === 'system');
|
|
15
|
-
|
|
15
|
+
if (!systemMessage || typeof systemMessage.content !== 'string')
|
|
16
|
+
return null;
|
|
17
|
+
return systemMessage.content;
|
|
16
18
|
}
|
|
17
19
|
// Names the cache file by system prompt only. Tools stay out on purpose: the
|
|
18
20
|
// addon compares the rendered prompt against the tokens the file holds, so a
|
|
@@ -37,7 +39,9 @@ export function generateCacheKey(messages, tools) {
|
|
|
37
39
|
const hashString = hash.digest('hex');
|
|
38
40
|
return hashString.substring(0, 16);
|
|
39
41
|
}
|
|
40
|
-
|
|
42
|
+
// Resolves the path without creating its directories, for a cache that may
|
|
43
|
+
// not exist (a seed source).
|
|
44
|
+
export function resolveCacheFilePath(modelId, configHash, cacheKey) {
|
|
41
45
|
const cacheDir = getKVCacheDir();
|
|
42
46
|
const sessionCacheDir = validateAndJoinPath(cacheDir, cacheKey);
|
|
43
47
|
const modelCacheDir = validateAndJoinPath(sessionCacheDir, modelId);
|
|
@@ -10,5 +10,7 @@ export interface LoadModelOptions {
|
|
|
10
10
|
signal: AbortSignal;
|
|
11
11
|
requestId: string;
|
|
12
12
|
};
|
|
13
|
+
/** Session that takes a lease on the model. Defaults to the owner session. */
|
|
14
|
+
sessionId?: string;
|
|
13
15
|
}
|
|
14
16
|
export declare function loadModel(params: LoadModelServerParams, options?: LoadModelOptions): Promise<LoadModelResult>;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { loadModelServerParamsSchema, normalizeModelType } from '../../schemas/index.js';
|
|
2
2
|
import { nowMs } from '../../profiling/clock.js';
|
|
3
3
|
import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.js';
|
|
4
|
-
import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
|
|
4
|
+
import { addModelLease, getModelEntry, isModelLoaded, isModelReconfigured, registerModel } from '../../runtime/model-registry.js';
|
|
5
|
+
import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
|
|
6
|
+
import { withModelWorkLock } from '../../runtime/model-work-lock.js';
|
|
5
7
|
import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
|
|
6
|
-
import { detectShardedModel,
|
|
7
|
-
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, InferenceCancelledError } from '../../errors/index.js';
|
|
8
|
+
import { detectShardedModel, checkAllShardsExist } from '../../utils/index.js';
|
|
9
|
+
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, ModelReconfiguredError, InferenceCancelledError } from '../../errors/index.js';
|
|
8
10
|
import { ensureAddonLoggerReady, getPlugin } from '../../plugins/index.js';
|
|
9
|
-
import {
|
|
11
|
+
import { runLoadFitCheck } from '../../model-fit/load-check.js';
|
|
10
12
|
import { refusesLoad, resolveModelFitPolicy, runsProbe } from '../../schemas/model-fit-policy.js';
|
|
11
13
|
import { getConfig } from '../../runtime/state.js';
|
|
12
14
|
import { resolveToolDialect } from '../../utils/tools/dialect.js';
|
|
@@ -15,13 +17,30 @@ import path from 'bare-path';
|
|
|
15
17
|
import { getEngineLogger } from '../../logging/index.js';
|
|
16
18
|
const logger = getEngineLogger();
|
|
17
19
|
export async function loadModel(params, options) {
|
|
18
|
-
const
|
|
20
|
+
const parsed = loadModelServerParamsSchema.parse(params);
|
|
21
|
+
return withModelWorkLock(parsed.modelId, () => loadModelLocked(parsed, options), options?.request);
|
|
22
|
+
}
|
|
23
|
+
async function loadModelLocked(params, options) {
|
|
24
|
+
const { modelId, modelPath, options: modelOptions, artifacts, modelName } = params;
|
|
19
25
|
const { modelConfig, modelType: rawModelType, modelFitPolicy } = modelOptions;
|
|
20
26
|
// Normalize modelType to canonical form (handles aliases and custom types)
|
|
21
27
|
const modelType = normalizeModelType(rawModelType);
|
|
22
|
-
|
|
28
|
+
const sessionId = options?.sessionId ?? OWNER_SESSION_ID;
|
|
29
|
+
// A cancel that landed while this load waited for another load of the same
|
|
30
|
+
// model must not take a lease the caller no longer wants.
|
|
31
|
+
if (options?.request?.signal.aborted === true) {
|
|
32
|
+
throw new InferenceCancelledError(options.request.requestId);
|
|
33
|
+
}
|
|
34
|
+
// An already loaded model is shared: the caller takes a lease on it. A
|
|
35
|
+
// session that holds no lease on a reconfigured model is refused: it asked
|
|
36
|
+
// for the configuration the id hashes, which the model no longer has.
|
|
23
37
|
if (isModelLoaded(modelId)) {
|
|
24
|
-
|
|
38
|
+
const entry = getModelEntry(modelId);
|
|
39
|
+
if (entry && !entry.leases.has(sessionId) && isModelReconfigured(modelId)) {
|
|
40
|
+
throw new ModelReconfiguredError(modelId);
|
|
41
|
+
}
|
|
42
|
+
addModelLease(modelId, sessionId);
|
|
43
|
+
logger.info(`${modelType} model ${modelId} is already loaded; lease added for ${sessionId}`);
|
|
25
44
|
return {};
|
|
26
45
|
}
|
|
27
46
|
// Detect if sharded model
|
|
@@ -33,12 +52,9 @@ export async function loadModel(params, options) {
|
|
|
33
52
|
throw new PluginNotFoundError(modelType);
|
|
34
53
|
}
|
|
35
54
|
if (isShardedModel) {
|
|
36
|
-
// For sharded models, validate all shards and tensors.txt exist
|
|
37
55
|
const shardDir = path.dirname(modelPath);
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const numberedShards = generateShardFilenames(modelFileName);
|
|
41
|
-
throw new ModelFileNotFoundError(`Missing shards or ${shardInfo.baseFilename}.tensors.txt. Expected ${numberedShards.length} shard files + tensors.txt in ${shardDir}`);
|
|
56
|
+
if (!(await checkAllShardsExist(shardDir, modelFileName))) {
|
|
57
|
+
throw new ModelFileNotFoundError(`Missing shards. Expected ${shardInfo.totalShards} shard files in ${shardDir}`);
|
|
42
58
|
}
|
|
43
59
|
}
|
|
44
60
|
else if (!plugin.skipPrimaryModelPathValidation) {
|
|
@@ -61,8 +77,8 @@ export async function loadModel(params, options) {
|
|
|
61
77
|
// the native load for device memory.
|
|
62
78
|
const fitPolicy = resolveModelFitPolicy(modelFitPolicy, getConfig().modelFitPolicy);
|
|
63
79
|
const request = options?.request;
|
|
64
|
-
const
|
|
65
|
-
? await
|
|
80
|
+
const fitCheck = runsProbe(fitPolicy)
|
|
81
|
+
? await runLoadFitCheck({
|
|
66
82
|
modelId,
|
|
67
83
|
modelType: modelType,
|
|
68
84
|
modelPath,
|
|
@@ -71,11 +87,12 @@ export async function loadModel(params, options) {
|
|
|
71
87
|
isShardedModel
|
|
72
88
|
}, request !== undefined ? { signal: request.signal } : {})
|
|
73
89
|
: undefined;
|
|
90
|
+
const fitProbe = fitCheck?.probe;
|
|
74
91
|
if (request?.signal.aborted === true) {
|
|
75
92
|
throw new InferenceCancelledError(request.requestId);
|
|
76
93
|
}
|
|
77
94
|
if (refusesLoad(fitPolicy, fitProbe?.verdict)) {
|
|
78
|
-
throw new ModelFitRefusedError(modelType,
|
|
95
|
+
throw new ModelFitRefusedError(modelType, 'does-not-fit', fitCheck?.detail);
|
|
79
96
|
}
|
|
80
97
|
// Load the addon's logger before the addon itself runs. Plugins that defer
|
|
81
98
|
// their logging module do so to keep registration free of native loads, so
|
|
@@ -106,7 +123,7 @@ export async function loadModel(params, options) {
|
|
|
106
123
|
name: modelName,
|
|
107
124
|
fitProbe,
|
|
108
125
|
toolDialect
|
|
109
|
-
});
|
|
126
|
+
}, sessionId);
|
|
110
127
|
const loadResult = modelInitializationTimeMs !== undefined ? { timing: { modelInitializationTimeMs } } : {};
|
|
111
128
|
return forwardBackendDiagnostics(loadResult, result);
|
|
112
129
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../schemas/plugin.js';
|
|
1
2
|
import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
|
|
2
3
|
import type Buffer from 'bare-buffer';
|
|
3
4
|
import type { InferenceBackendDiagnostics } from '../../schemas/index.js';
|
|
@@ -10,13 +11,13 @@ type TranscribeReturn = {
|
|
|
10
11
|
};
|
|
11
12
|
export declare function transcribe(params: TranscribeParams & {
|
|
12
13
|
metadata: true;
|
|
13
|
-
}, requestId?: string): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
|
|
14
|
-
export declare function transcribe(params: TranscribeParams, requestId?: string): AsyncGenerator<string, TranscribeReturn, void>;
|
|
14
|
+
}, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
|
|
15
|
+
export declare function transcribe(params: TranscribeParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, TranscribeReturn, void>;
|
|
15
16
|
export interface TranscribeStreamOpts {
|
|
16
17
|
emitVadEvents?: boolean;
|
|
17
18
|
endOfTurnSilenceMs?: number;
|
|
18
19
|
vadRunIntervalMs?: number;
|
|
19
20
|
parakeetStreamingConfig?: ParakeetStreamingRunConfig;
|
|
20
21
|
}
|
|
21
|
-
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
22
|
-
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
22
|
+
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
23
|
+
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
|