@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -8,7 +8,9 @@ import { diffusion } from '../../../plugins/builtin/sdcpp-generation/ops/diffusi
|
|
|
8
8
|
import { markLtxVideoModel, markH3VideoModel, markMoeCapableVideoModel, video } from '../../../plugins/builtin/sdcpp-generation/ops/video.js';
|
|
9
9
|
import { upscale } from '../../../plugins/builtin/sdcpp-generation/ops/upscale.js';
|
|
10
10
|
import { createWorldSession, worldCreateScene, worldScenePath, worldStep } from '../../../plugins/builtin/sdcpp-generation/ops/world.js';
|
|
11
|
-
import {
|
|
11
|
+
import { sdcppFitRequest } from '../../../plugins/builtin/sdcpp-generation/fit.js';
|
|
12
|
+
import { assertUpscalerResolved, imageConfigFor, imageFilesFor, videoConfigFor, videoFilesFor } from '../../../plugins/builtin/sdcpp-generation/load-files.js';
|
|
13
|
+
import { toEsrganAddonConfig } from '../../../plugins/builtin/sdcpp-generation/upscaler-config.js';
|
|
12
14
|
import { splitWorldConfig } from '../../../plugins/builtin/sdcpp-generation/world-config.js';
|
|
13
15
|
// Wait for op cleanup before emitting completion.
|
|
14
16
|
async function* withCompletionFrame(stream, complete) {
|
|
@@ -35,6 +37,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
35
37
|
displayName: 'Image Generation & Upscaling (stable-diffusion.cpp)',
|
|
36
38
|
addonPackage: ADDON_DIFFUSION,
|
|
37
39
|
loadConfigSchema: sdcppConfigSchema,
|
|
40
|
+
fitRequest: sdcppFitRequest,
|
|
38
41
|
assessFit: diffusionAssessFit,
|
|
39
42
|
async resolveConfig(cfg, ctx) {
|
|
40
43
|
if (cfg.projectionModelSrc && cfg.mode !== 'diffusion') {
|
|
@@ -52,6 +55,14 @@ export const diffusionPlugin = definePlugin({
|
|
|
52
55
|
throw new ModelLoadFailedError('modelConfig.uncondModelSrc selects the Ideogram 4 layout and requires ' +
|
|
53
56
|
'modelConfig.llmModelSrc (Qwen3-VL) and modelConfig.vaeModelSrc.');
|
|
54
57
|
}
|
|
58
|
+
if (cfg.tokenizerSrc && cfg.mode !== 'diffusion') {
|
|
59
|
+
throw new ModelLoadFailedError('modelConfig.tokenizerSrc is MOSS-Image diffusion only. ' +
|
|
60
|
+
"Use mode: 'diffusion' or remove tokenizerSrc.");
|
|
61
|
+
}
|
|
62
|
+
if (cfg.tokenizerSrc && (!cfg.llmModelSrc || !cfg.vaeModelSrc)) {
|
|
63
|
+
throw new ModelLoadFailedError('modelConfig.tokenizerSrc selects the MOSS-Image layout and requires ' +
|
|
64
|
+
'modelConfig.llmModelSrc (Qwen3.5) and modelConfig.vaeModelSrc.');
|
|
65
|
+
}
|
|
55
66
|
// A high-noise expert only has somewhere to go in the Wan 2.2 A14B video
|
|
56
67
|
// layout: every other mode builds a model with no second-expert slot, and
|
|
57
68
|
// embeddingsConnectorsModelSrc selects LTX-2 instead. Check it ahead of the
|
|
@@ -140,7 +151,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
140
151
|
if (cfg.mode === 'upscale') {
|
|
141
152
|
return { config: cfg };
|
|
142
153
|
}
|
|
143
|
-
const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, projectionModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, taehvModelSrc, sceneSrc, upscaler, ...rest } = cfg;
|
|
154
|
+
const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, projectionModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, tokenizerSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, taehvModelSrc, sceneSrc, upscaler, ...rest } = cfg;
|
|
144
155
|
if (embeddingsConnectorsModelSrc && cfg.mode !== 'video') {
|
|
145
156
|
throw new ModelLoadFailedError('modelConfig.embeddingsConnectorsModelSrc selects the LTX-2 video ' +
|
|
146
157
|
"layout and is only valid with mode: 'video'.");
|
|
@@ -184,6 +195,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
184
195
|
vaeModelSrc,
|
|
185
196
|
highNoiseDiffusionModelSrc,
|
|
186
197
|
uncondModelSrc,
|
|
198
|
+
tokenizerSrc,
|
|
187
199
|
audioVaeModelSrc,
|
|
188
200
|
embeddingsConnectorsModelSrc,
|
|
189
201
|
esrganModelSrc,
|
|
@@ -195,7 +207,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
195
207
|
return { config: runtimeConfig };
|
|
196
208
|
}
|
|
197
209
|
const resolve = ctx.resolveModelPath;
|
|
198
|
-
const [clipLModelPath, clipGModelPath, clipVisionModelPath, t5XxlModelPath, llmModelPath, projectionModelPath, vaeModelPath, highNoiseDiffusionModelPath, uncondModelPath, audioVaeModelPath, embeddingsConnectorsModelPath, esrganModelPath, taehvModelPath, seedScenePath] = await Promise.all([
|
|
210
|
+
const [clipLModelPath, clipGModelPath, clipVisionModelPath, t5XxlModelPath, llmModelPath, projectionModelPath, vaeModelPath, highNoiseDiffusionModelPath, uncondModelPath, tokenizerPath, audioVaeModelPath, embeddingsConnectorsModelPath, esrganModelPath, taehvModelPath, seedScenePath] = await Promise.all([
|
|
199
211
|
clipLModelSrc ? resolve(clipLModelSrc) : undefined,
|
|
200
212
|
clipGModelSrc ? resolve(clipGModelSrc) : undefined,
|
|
201
213
|
clipVisionModelSrc ? resolve(clipVisionModelSrc) : undefined,
|
|
@@ -205,6 +217,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
205
217
|
vaeModelSrc ? resolve(vaeModelSrc) : undefined,
|
|
206
218
|
highNoiseDiffusionModelSrc ? resolve(highNoiseDiffusionModelSrc) : undefined,
|
|
207
219
|
uncondModelSrc ? resolve(uncondModelSrc) : undefined,
|
|
220
|
+
tokenizerSrc ? resolve(tokenizerSrc) : undefined,
|
|
208
221
|
audioVaeModelSrc ? resolve(audioVaeModelSrc) : undefined,
|
|
209
222
|
embeddingsConnectorsModelSrc ? resolve(embeddingsConnectorsModelSrc) : undefined,
|
|
210
223
|
esrganModelSrc ? resolve(esrganModelSrc) : undefined,
|
|
@@ -223,6 +236,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
223
236
|
...(vaeModelPath && { vaeModelPath }),
|
|
224
237
|
...(highNoiseDiffusionModelPath && { highNoiseDiffusionModelPath }),
|
|
225
238
|
...(uncondModelPath && { uncondModelPath }),
|
|
239
|
+
...(tokenizerPath && { tokenizerPath }),
|
|
226
240
|
...(audioVaeModelPath && { audioVaeModelPath }),
|
|
227
241
|
...(embeddingsConnectorsModelPath && { embeddingsConnectorsModelPath }),
|
|
228
242
|
...(esrganModelPath && { esrganModelPath }),
|
|
@@ -241,19 +255,10 @@ export const diffusionPlugin = definePlugin({
|
|
|
241
255
|
getEngineLogger().warn(`[${ModelType.sdcppGeneration}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
|
|
242
256
|
}
|
|
243
257
|
}
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
|
|
248
|
-
// logger / native side-effects so callers can recover cleanly.
|
|
249
|
-
if (config.mode === 'diffusion' &&
|
|
250
|
-
config.upscaler !== undefined &&
|
|
251
|
-
!artifacts?.['esrganModelPath']) {
|
|
252
|
-
throw new ModelLoadFailedError('modelConfig.upscaler.model_src is required when modelConfig.upscaler ' +
|
|
253
|
-
'is set in diffusion mode. Provide the ESRGAN model, omit the ' +
|
|
254
|
-
"upscaler block, or switch to modelConfig.mode = 'upscale' to load " +
|
|
255
|
-
'a standalone upscaler.');
|
|
256
|
-
}
|
|
258
|
+
// Reject early with a clear error instead of letting the native addon fail
|
|
259
|
+
// mid-load. Done before any logger / native side-effects so callers can
|
|
260
|
+
// recover cleanly.
|
|
261
|
+
assertUpscalerResolved(config, artifacts);
|
|
257
262
|
const logger = createStreamLogger(modelId, ModelType.sdcppGeneration);
|
|
258
263
|
registerAddonLogger(modelId, ModelType.sdcppGeneration, logger);
|
|
259
264
|
if (config.mode === 'upscale') {
|
|
@@ -292,86 +297,14 @@ export const diffusionPlugin = definePlugin({
|
|
|
292
297
|
return { model };
|
|
293
298
|
}
|
|
294
299
|
if (config.mode === 'video') {
|
|
295
|
-
|
|
296
|
-
// the LTX-2 text-embedding connectors switch to the LTX-2 layout (Gemma
|
|
297
|
-
// via llm + video VAE + connectors, optional audio VAE —
|
|
298
|
-
// `SdModel::isLtxModel_ = !embeddingsConnectorsPath.empty()`); without
|
|
299
|
-
// them, an audio VAE (or an llm with no Wan T5) means MiniMax-H3
|
|
300
|
-
// (llm + video VAE, with an optional audio VAE); everything else is Wan
|
|
301
|
-
// via t5Xxl.
|
|
302
|
-
// Native code verifies the model's tensors.
|
|
303
|
-
const embeddingsConnectorsModelPath = artifacts?.['embeddingsConnectorsModelPath'];
|
|
304
|
-
if (!artifacts?.['vaeModelPath']) {
|
|
305
|
-
throw new ModelLoadFailedError('modelConfig.vaeModelSrc is required in video mode. ' +
|
|
306
|
-
'Provide the Wan, LTX-2 or MiniMax-H3 video VAE before loading the video pipeline.');
|
|
307
|
-
}
|
|
308
|
-
const vaeModelPath = artifacts['vaeModelPath'];
|
|
309
|
-
let files;
|
|
310
|
-
if (embeddingsConnectorsModelPath) {
|
|
311
|
-
if (!artifacts['llmModelPath']) {
|
|
312
|
-
throw new ModelLoadFailedError('modelConfig.llmModelSrc is required for LTX-2 video. ' +
|
|
313
|
-
'Provide the Gemma text encoder model before loading the LTX-2 pipeline.');
|
|
314
|
-
}
|
|
315
|
-
files = {
|
|
316
|
-
model: modelPath,
|
|
317
|
-
vae: vaeModelPath,
|
|
318
|
-
llm: artifacts['llmModelPath'],
|
|
319
|
-
embeddingsConnectors: embeddingsConnectorsModelPath,
|
|
320
|
-
...(artifacts['audioVaeModelPath'] && {
|
|
321
|
-
audioVae: artifacts['audioVaeModelPath']
|
|
322
|
-
}),
|
|
323
|
-
...(artifacts['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
|
|
324
|
-
};
|
|
325
|
-
}
|
|
326
|
-
else if (artifacts['audioVaeModelPath'] ||
|
|
327
|
-
(artifacts['llmModelPath'] && !artifacts['t5XxlModelPath'])) {
|
|
328
|
-
if (!artifacts['llmModelPath']) {
|
|
329
|
-
throw new ModelLoadFailedError('MiniMax-H3 requires llmModelSrc and vaeModelSrc.');
|
|
330
|
-
}
|
|
331
|
-
if (artifacts['t5XxlModelPath'] ||
|
|
332
|
-
artifacts['highNoiseDiffusionModelPath'] ||
|
|
333
|
-
artifacts['clipVisionModelPath'] ||
|
|
334
|
-
artifacts['clipLModelPath'] ||
|
|
335
|
-
artifacts['clipGModelPath']) {
|
|
336
|
-
throw new ModelLoadFailedError('MiniMax-H3 cannot use Wan T5, high-noise, CLIP-vision or CLIP-L/G companions.');
|
|
337
|
-
}
|
|
338
|
-
files = {
|
|
339
|
-
model: modelPath,
|
|
340
|
-
vae: vaeModelPath,
|
|
341
|
-
llm: artifacts['llmModelPath'],
|
|
342
|
-
...(artifacts['audioVaeModelPath'] && {
|
|
343
|
-
audioVae: artifacts['audioVaeModelPath']
|
|
344
|
-
})
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
else {
|
|
348
|
-
if (!artifacts['t5XxlModelPath']) {
|
|
349
|
-
throw new ModelLoadFailedError('modelConfig.t5XxlModelSrc is required in video mode. ' +
|
|
350
|
-
'Provide the Wan text encoder model before loading the video pipeline.');
|
|
351
|
-
}
|
|
352
|
-
files = {
|
|
353
|
-
model: modelPath,
|
|
354
|
-
vae: vaeModelPath,
|
|
355
|
-
t5Xxl: artifacts['t5XxlModelPath'],
|
|
356
|
-
...(artifacts['highNoiseDiffusionModelPath'] && {
|
|
357
|
-
highNoiseDiffusionModel: artifacts['highNoiseDiffusionModelPath']
|
|
358
|
-
}),
|
|
359
|
-
...(artifacts['clipVisionModelPath'] && {
|
|
360
|
-
clipVision: artifacts['clipVisionModelPath']
|
|
361
|
-
}),
|
|
362
|
-
...(artifacts['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
|
|
363
|
-
};
|
|
364
|
-
}
|
|
365
|
-
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
366
|
-
const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, projectionModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, upscaler, mode, ...rest } = config;
|
|
367
|
-
/* eslint-enable @typescript-eslint/no-unused-vars */
|
|
300
|
+
const files = videoFilesFor(modelPath, artifacts);
|
|
368
301
|
const model = new VideoStableDiffusion({
|
|
369
302
|
files,
|
|
370
|
-
config:
|
|
303
|
+
config: videoConfigFor(config),
|
|
371
304
|
logger,
|
|
372
305
|
opts: { stats: true }
|
|
373
306
|
});
|
|
374
|
-
if (
|
|
307
|
+
if (files.embeddingsConnectors)
|
|
375
308
|
markLtxVideoModel(model);
|
|
376
309
|
else if (files.llm)
|
|
377
310
|
markH3VideoModel(model);
|
|
@@ -379,31 +312,9 @@ export const diffusionPlugin = definePlugin({
|
|
|
379
312
|
markMoeCapableVideoModel(model);
|
|
380
313
|
return { model };
|
|
381
314
|
}
|
|
382
|
-
const files = {
|
|
383
|
-
model: modelPath,
|
|
384
|
-
...(artifacts?.['clipLModelPath'] && { clipL: artifacts['clipLModelPath'] }),
|
|
385
|
-
...(artifacts?.['clipGModelPath'] && { clipG: artifacts['clipGModelPath'] }),
|
|
386
|
-
...(artifacts?.['t5XxlModelPath'] && { t5Xxl: artifacts['t5XxlModelPath'] }),
|
|
387
|
-
...(artifacts?.['llmModelPath'] && { llm: artifacts['llmModelPath'] }),
|
|
388
|
-
...(artifacts?.['projectionModelPath'] && { llmVision: artifacts['projectionModelPath'] }),
|
|
389
|
-
...(artifacts?.['vaeModelPath'] && { vae: artifacts['vaeModelPath'] }),
|
|
390
|
-
...(artifacts?.['uncondModelPath'] && {
|
|
391
|
-
uncondModel: artifacts['uncondModelPath']
|
|
392
|
-
}),
|
|
393
|
-
...(artifacts?.['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
|
|
394
|
-
};
|
|
395
|
-
// `mode` is consumed by this plugin to select the model class above; the
|
|
396
|
-
// stable-diffusion.cpp native config does not understand it. Strip it
|
|
397
|
-
// before forwarding `rest` to the addon.
|
|
398
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
399
|
-
const { upscaler, mode, ...rest } = config;
|
|
400
|
-
const addonConfig = {
|
|
401
|
-
...rest,
|
|
402
|
-
...flattenUpscalerKeys(upscaler)
|
|
403
|
-
};
|
|
404
315
|
const model = new ImgStableDiffusion({
|
|
405
|
-
files,
|
|
406
|
-
config:
|
|
316
|
+
files: imageFilesFor(modelPath, artifacts),
|
|
317
|
+
config: imageConfigFor(config),
|
|
407
318
|
logger,
|
|
408
319
|
opts: { stats: true }
|
|
409
320
|
});
|
|
@@ -417,8 +328,9 @@ export const diffusionPlugin = definePlugin({
|
|
|
417
328
|
// sdcpp diffusion exposes a model-wide hard cancel — compute
|
|
418
329
|
// is interrupted on the currently-running generation.
|
|
419
330
|
cancel: { scope: 'model', hard: true },
|
|
420
|
-
handler: (request) => withCompletionFrame(diffusion(request), ({ stats }) => ({
|
|
331
|
+
handler: (request, _inputStream, context) => withCompletionFrame(diffusion(request, context), ({ requestId, stats }) => ({
|
|
421
332
|
type: 'diffusionStream',
|
|
333
|
+
requestId,
|
|
422
334
|
done: true,
|
|
423
335
|
stats
|
|
424
336
|
}))
|
|
@@ -428,7 +340,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
428
340
|
responseSchema: videoStreamResponseSchema,
|
|
429
341
|
streaming: true,
|
|
430
342
|
cancel: { scope: 'model', hard: true },
|
|
431
|
-
handler: (request) => withCompletionFrame(video(request), ({ stats }) => ({
|
|
343
|
+
handler: (request, _inputStream, context) => withCompletionFrame(video(request, context), ({ stats }) => ({
|
|
432
344
|
type: 'videoStream',
|
|
433
345
|
done: true,
|
|
434
346
|
stats
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import fs from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { gpuLayersFromCount } from '../../../model-fit/gpu.js';
|
|
4
|
+
const COSYVOICE_COMPANIONS_MISSING = 'the cosyvoice companion set is not laid out beside the checkpoint';
|
|
5
|
+
/**
|
|
6
|
+
* CosyVoice's companion set co-locates the flow, HiFT and voice GGUFs beside
|
|
7
|
+
* the LLM checkpoint, and the loader finds them by name in that directory. The
|
|
8
|
+
* fitter takes explicit paths, so the same directory is read here.
|
|
9
|
+
*/
|
|
10
|
+
function cosyvoiceRequest(modelPath, common) {
|
|
11
|
+
const dir = path.dirname(modelPath);
|
|
12
|
+
let entries;
|
|
13
|
+
try {
|
|
14
|
+
entries = fs.readdirSync(dir);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return { unknown: COSYVOICE_COMPANIONS_MISSING };
|
|
18
|
+
}
|
|
19
|
+
const companion = (matches) => {
|
|
20
|
+
const found = entries.find(matches);
|
|
21
|
+
return found === undefined ? undefined : path.join(dir, found);
|
|
22
|
+
};
|
|
23
|
+
const flowPath = companion((name) => name.includes('-flow-'));
|
|
24
|
+
const hiftPath = companion((name) => name.includes('-hift-'));
|
|
25
|
+
const voicePath = companion((name) => name === 'voice.gguf');
|
|
26
|
+
if (flowPath === undefined || hiftPath === undefined || voicePath === undefined) {
|
|
27
|
+
return { unknown: COSYVOICE_COMPANIONS_MISSING };
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
request: {
|
|
31
|
+
engineType: 'cosyvoice3',
|
|
32
|
+
cosyvoiceLlmModelPath: modelPath,
|
|
33
|
+
cosyvoiceFlowModelPath: flowPath,
|
|
34
|
+
cosyvoiceHiftModelPath: hiftPath,
|
|
35
|
+
cosyvoiceVoiceModelPath: voicePath,
|
|
36
|
+
...common
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/** The tts-ggml fit request for a load, built for the voice engine it names. */
|
|
41
|
+
export function ttsFitRequest(load) {
|
|
42
|
+
const config = (load.modelConfig ?? {});
|
|
43
|
+
const artifacts = load.artifacts ?? {};
|
|
44
|
+
const workload = load.ttsWorkload;
|
|
45
|
+
// The enhancer and denoiser load next to the voice, and no fitter sizes them.
|
|
46
|
+
if (artifacts['lavasrEnhancerPath'] !== undefined ||
|
|
47
|
+
artifacts['lavasrDenoiserPath'] !== undefined) {
|
|
48
|
+
return { unknown: 'the LavaSR stages have no fitter' };
|
|
49
|
+
}
|
|
50
|
+
// The explicit backend selector resolves the same device the load would, so
|
|
51
|
+
// the projection lands on that device rather than on the `useGPU` policy's.
|
|
52
|
+
const common = {
|
|
53
|
+
nGpuLayers: gpuLayersFromCount(config.useGPU, config.nGpuLayers),
|
|
54
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
55
|
+
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir })
|
|
56
|
+
};
|
|
57
|
+
const request = (fit) => ({ request: fit });
|
|
58
|
+
switch (config.ttsEngine) {
|
|
59
|
+
case 'supertonic':
|
|
60
|
+
// Precision and the f16 weight override change the weight footprint the
|
|
61
|
+
// fitter sizes, so they travel with the request.
|
|
62
|
+
return request({
|
|
63
|
+
engineType: 'supertonic',
|
|
64
|
+
supertonicModelPath: load.modelPath,
|
|
65
|
+
...common,
|
|
66
|
+
...(config.precision !== undefined && { precision: config.precision }),
|
|
67
|
+
...(config.f16Weights !== undefined && { f16Weights: config.f16Weights }),
|
|
68
|
+
...(config.ttsNumInferenceSteps !== undefined && { steps: config.ttsNumInferenceSteps })
|
|
69
|
+
});
|
|
70
|
+
case 'parler':
|
|
71
|
+
// The decoder sidecar policy decides whether the decoder weights load on
|
|
72
|
+
// ggml at all; the fitter matches the load's choice.
|
|
73
|
+
return request({
|
|
74
|
+
engineType: 'parler',
|
|
75
|
+
parlerModelPath: load.modelPath,
|
|
76
|
+
...common,
|
|
77
|
+
...(config.coremlDecoder !== undefined && { coremlDecoder: config.coremlDecoder }),
|
|
78
|
+
...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
|
|
79
|
+
});
|
|
80
|
+
case 'audio8': {
|
|
81
|
+
const codecDecoderPath = artifacts['audio8CodecDecoderPath'];
|
|
82
|
+
if (codecDecoderPath === undefined) {
|
|
83
|
+
return { unknown: 'the audio8 codec decoder is not resolved' };
|
|
84
|
+
}
|
|
85
|
+
const codecEncoderPath = artifacts['audio8CodecEncoderPath'];
|
|
86
|
+
return request({
|
|
87
|
+
engineType: 'audio8',
|
|
88
|
+
audio8LmPath: load.modelPath,
|
|
89
|
+
audio8CodecDecoderPath: codecDecoderPath,
|
|
90
|
+
...(codecEncoderPath !== undefined && { audio8CodecEncoderPath: codecEncoderPath }),
|
|
91
|
+
...common,
|
|
92
|
+
...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
case 'cosyvoice3':
|
|
96
|
+
return cosyvoiceRequest(load.modelPath, common);
|
|
97
|
+
case 'moss': {
|
|
98
|
+
const decoder = artifacts['mossCodecDecoderPath'];
|
|
99
|
+
const encoder = artifacts['mossCodecEncoderPath'];
|
|
100
|
+
if (decoder === undefined)
|
|
101
|
+
return { unknown: 'the moss codec decoder is not resolved' };
|
|
102
|
+
if (workload === undefined || !('promptRows' in workload)) {
|
|
103
|
+
return {
|
|
104
|
+
unknown: 'a MOSS projection needs a ttsWorkload with promptRows, referenceSamples and streaming'
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
if (workload.referenceSamples > 0 && encoder === undefined) {
|
|
108
|
+
return { unknown: 'MOSS reference encoding needs the codec encoder' };
|
|
109
|
+
}
|
|
110
|
+
return request({
|
|
111
|
+
engineType: 'moss',
|
|
112
|
+
mossBackbonePath: load.modelPath,
|
|
113
|
+
mossCodecDecoderPath: decoder,
|
|
114
|
+
...(encoder !== undefined && { mossCodecEncoderPath: encoder }),
|
|
115
|
+
...common,
|
|
116
|
+
...workload,
|
|
117
|
+
...(config.threads !== undefined && { threads: config.threads }),
|
|
118
|
+
...(config.streamChunkTokens !== undefined && {
|
|
119
|
+
streamChunkTokens: config.streamChunkTokens
|
|
120
|
+
}),
|
|
121
|
+
...(config.durationTokens !== undefined && { durationTokens: config.durationTokens }),
|
|
122
|
+
// The context and generation budgets size the KV cache and the
|
|
123
|
+
// generation buffers the fitter projects.
|
|
124
|
+
...(config.nCtx !== undefined && { nCtx: config.nCtx }),
|
|
125
|
+
...(config.maxNewTokens !== undefined && { maxNewTokens: config.maxNewTokens })
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
case 'moss-sfx':
|
|
129
|
+
if (workload === undefined || !('prompt' in workload)) {
|
|
130
|
+
return {
|
|
131
|
+
unknown: 'a MOSS-SoundEffect projection needs a ttsWorkload with prompt and seconds'
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
return request({
|
|
135
|
+
engineType: 'moss-sfx',
|
|
136
|
+
mossSoundEffectPath: load.modelPath,
|
|
137
|
+
...common,
|
|
138
|
+
...workload,
|
|
139
|
+
...(config.negativePrompt !== undefined && { negativePrompt: config.negativePrompt }),
|
|
140
|
+
...(config.steps !== undefined && { steps: config.steps }),
|
|
141
|
+
...(config.guidance !== undefined && { guidance: config.guidance }),
|
|
142
|
+
...(config.shift !== undefined && { shift: config.shift }),
|
|
143
|
+
...(config.threads !== undefined && { threads: config.threads })
|
|
144
|
+
});
|
|
145
|
+
case 'moss-speech':
|
|
146
|
+
return { unknown: 'MOSS-Speech has no fitter' };
|
|
147
|
+
default: {
|
|
148
|
+
const s3genPath = artifacts['s3genPath'];
|
|
149
|
+
if (s3genPath === undefined) {
|
|
150
|
+
return { unknown: 'the chatterbox s3gen checkpoint is not resolved' };
|
|
151
|
+
}
|
|
152
|
+
return request({
|
|
153
|
+
engineType: 'chatterbox',
|
|
154
|
+
t3ModelPath: load.modelPath,
|
|
155
|
+
s3genModelPath: s3genPath,
|
|
156
|
+
...common,
|
|
157
|
+
...(config.nCtx !== undefined && config.nCtx > 0 && { nCtx: config.nCtx }),
|
|
158
|
+
...(config.kvCacheType !== undefined && { kvCacheType: config.kvCacheType })
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
2
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
3
|
/**
|
|
3
4
|
* The abort is persistent state, not a one-shot poke. The addon's `cancel()`
|
|
@@ -26,4 +27,4 @@ export declare function cancelIfAborted(model: unknown, signal: AbortSignal): Pr
|
|
|
26
27
|
* never owned the model slot, so cancelling then would interrupt the run that
|
|
27
28
|
* does.
|
|
28
29
|
*/
|
|
29
|
-
export declare function bindTtsCancel(model: unknown, modelId: string, requestId?: string): Promise<import("../../../../runtime/index.js").ManagedRequestContext>;
|
|
30
|
+
export declare function bindTtsCancel(model: unknown, modelId: string, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<import("../../../../runtime/index.js").ManagedRequestContext>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { requestParent } from '../../../../schemas/plugin.js';
|
|
1
2
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
2
3
|
import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
|
|
3
4
|
import { generateRequestId } from '../../../../runtime/request-id.js';
|
|
@@ -36,9 +37,11 @@ export async function cancelIfAborted(model, signal) {
|
|
|
36
37
|
* never owned the model slot, so cancelling then would interrupt the run that
|
|
37
38
|
* does.
|
|
38
39
|
*/
|
|
39
|
-
export async function bindTtsCancel(model, modelId, requestId) {
|
|
40
|
+
export async function bindTtsCancel(model, modelId, requestId, sessionId, context) {
|
|
40
41
|
const ctx = await getRequestRegistry().begin({
|
|
41
42
|
requestId: requestId ?? generateRequestId(),
|
|
43
|
+
sessionId,
|
|
44
|
+
...requestParent(context),
|
|
42
45
|
kind: 'tts',
|
|
43
46
|
modelId
|
|
44
47
|
});
|
|
@@ -15,7 +15,7 @@ export declare function getMossJobOptions(model: unknown, request: TtsRequest):
|
|
|
15
15
|
sampleRate?: number;
|
|
16
16
|
audio?: Float32Array<ArrayBuffer>;
|
|
17
17
|
text?: string;
|
|
18
|
-
role: "
|
|
18
|
+
role: "system" | "assistant" | "user";
|
|
19
19
|
}[];
|
|
20
20
|
sampleRate?: number;
|
|
21
21
|
audio?: Float32Array<ArrayBuffer>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
2
|
import { type TextToSpeechStreamRequest } from '../../../../schemas/index.js';
|
|
2
3
|
import Buffer from 'bare-buffer';
|
|
3
4
|
import { type TtsOpYield } from '../../../../utils/tts-stats.js';
|
|
4
5
|
import type { TtsOpResult } from '../../../../plugins/builtin/tts-ggml/ops/text-to-speech.js';
|
|
5
|
-
export declare function textToSpeechStream(params: TextToSpeechStreamRequest, inputStream: AsyncIterable<Buffer
|
|
6
|
+
export declare function textToSpeechStream(params: TextToSpeechStreamRequest, inputStream: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<TtsOpYield, TtsOpResult, unknown>;
|
|
@@ -56,9 +56,10 @@ import Buffer from 'bare-buffer';
|
|
|
56
56
|
import { nowMs } from '../../../../profiling/index.js';
|
|
57
57
|
import { buildStreamResult, hasDefinedValues } from '../../../../profiling/model-execution.js';
|
|
58
58
|
import { TextToSpeechStreamFailedError } from '../../../../errors/index.js';
|
|
59
|
-
import { collectTtsStats, chunkMetadata } from '../../../../utils/tts-stats.js';
|
|
59
|
+
import { carriesSpeechTokenIds, collectTtsStats, pcmFrame, chunkMetadata } from '../../../../utils/tts-stats.js';
|
|
60
60
|
import { assertParlerJobOptionsSupported, getParlerJobOptions, getTtsEngineType } from '../../../../plugins/builtin/tts-ggml/ops/parler-options.js';
|
|
61
61
|
import { bindTtsCancel, cancelIfAborted } from '../../../../plugins/builtin/tts-ggml/ops/cancel-binding.js';
|
|
62
|
+
import { assertVoiceJobOptionsSupported, resolveVoiceJobOptions } from '../../../../plugins/builtin/tts-ggml/ops/voice-options.js';
|
|
62
63
|
// Ends the text source the moment the run is cancelled. Without this an idle
|
|
63
64
|
// duplex session — cancelled while the accumulator is waiting for the next
|
|
64
65
|
// fragment, with no native job live — would sit in `runStreaming` holding the
|
|
@@ -149,7 +150,7 @@ async function* buffersToUtf8Fragments(inputStream) {
|
|
|
149
150
|
}
|
|
150
151
|
}
|
|
151
152
|
}
|
|
152
|
-
function buildRunStreamingOptions(request, parlerJobOptions) {
|
|
153
|
+
function buildRunStreamingOptions(request, parlerJobOptions, voiceJobOptions) {
|
|
153
154
|
const o = {};
|
|
154
155
|
if (request.accumulateSentences !== undefined) {
|
|
155
156
|
o['accumulateSentences'] = request.accumulateSentences;
|
|
@@ -163,9 +164,9 @@ function buildRunStreamingOptions(request, parlerJobOptions) {
|
|
|
163
164
|
if (request.flushAfterMs !== undefined) {
|
|
164
165
|
o['flushAfterMs'] = request.flushAfterMs;
|
|
165
166
|
}
|
|
166
|
-
return { ...o, ...parlerJobOptions };
|
|
167
|
+
return { ...o, ...parlerJobOptions, ...voiceJobOptions };
|
|
167
168
|
}
|
|
168
|
-
export async function* textToSpeechStream(params, inputStream) {
|
|
169
|
+
export async function* textToSpeechStream(params, inputStream, context) {
|
|
169
170
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
170
171
|
try {
|
|
171
172
|
const request = textToSpeechStreamRequestSchema.parse(params);
|
|
@@ -175,15 +176,26 @@ export async function* textToSpeechStream(params, inputStream) {
|
|
|
175
176
|
}
|
|
176
177
|
const parlerJobOptions = getParlerJobOptions(request);
|
|
177
178
|
assertParlerJobOptionsSupported(model, parlerJobOptions, 'textToSpeechStream');
|
|
178
|
-
|
|
179
|
+
assertVoiceJobOptionsSupported(model, request, 'textToSpeechStream');
|
|
180
|
+
const ctx = __addDisposableResource(env_1, await bindTtsCancel(model, request.modelId, request.requestId, request.sessionId, context), true);
|
|
179
181
|
if (ctx.signal.aborted)
|
|
180
182
|
return { ...buildStreamResult(0), cancelled: true };
|
|
183
|
+
// Resolved inside the request context: an uncached remote reference
|
|
184
|
+
// recording downloads here, and a cancel has to be able to stop that.
|
|
185
|
+
const voiceJobOptions = await resolveVoiceJobOptions(request, {
|
|
186
|
+
signal: ctx.signal,
|
|
187
|
+
scope: ctx.scope,
|
|
188
|
+
requestId: ctx.requestId,
|
|
189
|
+
sessionId: ctx.sessionId
|
|
190
|
+
});
|
|
191
|
+
if (voiceJobOptions === null)
|
|
192
|
+
return { ...buildStreamResult(0), cancelled: true };
|
|
181
193
|
const modelStart = nowMs();
|
|
182
194
|
if (!hasRunStreaming(model)) {
|
|
183
195
|
throw new TextToSpeechStreamFailedError('textToSpeechStream requires a TTS model with runStreaming');
|
|
184
196
|
}
|
|
185
197
|
const textSource = untilAborted(buffersToUtf8Fragments(inputStream), ctx.signal);
|
|
186
|
-
const streamOpts = buildRunStreamingOptions(request, parlerJobOptions);
|
|
198
|
+
const streamOpts = buildRunStreamingOptions(request, parlerJobOptions, voiceJobOptions);
|
|
187
199
|
const response = await model.runStreaming(textSource, Object.keys(streamOpts).length > 0 ? streamOpts : undefined);
|
|
188
200
|
try {
|
|
189
201
|
for await (const data of response.iterate()) {
|
|
@@ -193,11 +205,12 @@ export async function* textToSpeechStream(params, inputStream) {
|
|
|
193
205
|
if (data.outputArray == null) {
|
|
194
206
|
continue;
|
|
195
207
|
}
|
|
196
|
-
|
|
197
|
-
|
|
208
|
+
// An empty chunk is a native end marker, unless it carries the sentence's
|
|
209
|
+
// speech tokens (CosyVoice3 native streaming), which the caller wants.
|
|
210
|
+
if (data.outputArray.length === 0 && !carriesSpeechTokenIds(data)) {
|
|
198
211
|
continue;
|
|
199
212
|
}
|
|
200
|
-
yield {
|
|
213
|
+
yield { ...pcmFrame(data.outputArray), ...chunkMetadata(data) };
|
|
201
214
|
}
|
|
202
215
|
}
|
|
203
216
|
catch (error) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
2
|
import { type TtsRequest, type TtsStats } from '../../../../schemas/index.js';
|
|
2
3
|
import { type TtsOpYield } from '../../../../utils/tts-stats.js';
|
|
3
4
|
/** What the handler turns into the terminal frame. */
|
|
@@ -6,4 +7,4 @@ export type TtsOpResult = {
|
|
|
6
7
|
stats?: TtsStats;
|
|
7
8
|
cancelled?: boolean;
|
|
8
9
|
};
|
|
9
|
-
export declare function textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYield, TtsOpResult>;
|
|
10
|
+
export declare function textToSpeech(params: TtsRequest, context?: PluginHandlerContext): AsyncGenerator<TtsOpYield, TtsOpResult>;
|