@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import type ASRGgml from '@qvac/asr-ggml';
|
|
2
2
|
import type { ParakeetConfig, WhisperConfig, MossTranscribeConfig } from '../../../schemas/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Artifact key under which the parakeet plugin's `resolveConfig` stores the
|
|
5
|
+
* resolved `speakerAttribution.modelSrc`; the addon wants that path, not the
|
|
6
|
+
* source it was resolved from.
|
|
7
|
+
*/
|
|
8
|
+
export declare const SPEAKER_ATTRIBUTION_MODEL_PATH = "speakerAttributionModelPath";
|
|
3
9
|
export declare function buildWhisperEngineConfig(config: WhisperConfig): {
|
|
4
10
|
miscConfig?: {
|
|
5
11
|
[k: string]: unknown;
|
|
@@ -10,11 +16,15 @@ export declare function buildWhisperEngineConfig(config: WhisperConfig): {
|
|
|
10
16
|
engine: "whisper";
|
|
11
17
|
whisperConfig: ASRGgml.WhisperConfig;
|
|
12
18
|
};
|
|
13
|
-
|
|
19
|
+
/**
|
|
20
|
+
* The addon-facing parakeet config. `speakerAttribution.modelSrc` is a model
|
|
21
|
+
* source the SDK resolves before the load; the addon takes the resolved path
|
|
22
|
+
* as `speakerAttribution.modelPath`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function buildParakeetConfig(config: ParakeetConfig, artifacts?: Record<string, string>): ASRGgml.ParakeetConfig;
|
|
25
|
+
export declare function buildParakeetEngineConfig(config: ParakeetConfig, artifacts?: Record<string, string>): {
|
|
14
26
|
engine: "parakeet";
|
|
15
|
-
parakeetConfig:
|
|
16
|
-
[k: string]: unknown;
|
|
17
|
-
};
|
|
27
|
+
parakeetConfig: import("node_modules/@qvac/asr-ggml/engines/parakeet/driver").ParakeetConfig;
|
|
18
28
|
};
|
|
19
29
|
export declare function buildWhisperReloadConfig(config: WhisperConfig): {
|
|
20
30
|
miscConfig?: {
|
|
@@ -24,6 +34,11 @@ export declare function buildWhisperReloadConfig(config: WhisperConfig): {
|
|
|
24
34
|
[k: string]: unknown;
|
|
25
35
|
};
|
|
26
36
|
};
|
|
37
|
+
/**
|
|
38
|
+
* A reload merges into the loaded config, so leaving `speakerAttribution` out
|
|
39
|
+
* keeps the diarization model the load resolved; the reload path has no
|
|
40
|
+
* artifacts to resolve a new source with.
|
|
41
|
+
*/
|
|
27
42
|
export declare function buildParakeetReloadConfig(config: ParakeetConfig): {
|
|
28
43
|
parakeetConfig: {
|
|
29
44
|
[k: string]: unknown;
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import { ModelLoadFailedError } from '../../../errors/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Artifact key under which the parakeet plugin's `resolveConfig` stores the
|
|
4
|
+
* resolved `speakerAttribution.modelSrc`; the addon wants that path, not the
|
|
5
|
+
* source it was resolved from.
|
|
6
|
+
*/
|
|
7
|
+
export const SPEAKER_ATTRIBUTION_MODEL_PATH = 'speakerAttributionModelPath';
|
|
1
8
|
function omitUndefined(config) {
|
|
2
9
|
return Object.fromEntries(Object.entries(config).filter(([, value]) => value !== undefined));
|
|
3
10
|
}
|
|
@@ -24,10 +31,33 @@ export function buildWhisperEngineConfig(config) {
|
|
|
24
31
|
...(config.miscConfig && { miscConfig: omitUndefined(config.miscConfig) })
|
|
25
32
|
};
|
|
26
33
|
}
|
|
27
|
-
|
|
34
|
+
/**
|
|
35
|
+
* The addon-facing parakeet config. `speakerAttribution.modelSrc` is a model
|
|
36
|
+
* source the SDK resolves before the load; the addon takes the resolved path
|
|
37
|
+
* as `speakerAttribution.modelPath`.
|
|
38
|
+
*/
|
|
39
|
+
export function buildParakeetConfig(config, artifacts) {
|
|
40
|
+
const { speakerAttribution, ...rest } = config;
|
|
41
|
+
const parakeetConfig = omitUndefined(rest);
|
|
42
|
+
if (speakerAttribution === undefined)
|
|
43
|
+
return parakeetConfig;
|
|
44
|
+
const modelPath = artifacts?.[SPEAKER_ATTRIBUTION_MODEL_PATH];
|
|
45
|
+
if (!modelPath) {
|
|
46
|
+
throw new ModelLoadFailedError('speakerAttribution.modelSrc was not resolved to a diarization model path');
|
|
47
|
+
}
|
|
48
|
+
const { modelSrc: _modelSrc, ...options } = speakerAttribution;
|
|
49
|
+
return {
|
|
50
|
+
...parakeetConfig,
|
|
51
|
+
speakerAttribution: {
|
|
52
|
+
modelPath,
|
|
53
|
+
...omitUndefined(options)
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export function buildParakeetEngineConfig(config, artifacts) {
|
|
28
58
|
return {
|
|
29
59
|
engine: 'parakeet',
|
|
30
|
-
parakeetConfig:
|
|
60
|
+
parakeetConfig: buildParakeetConfig(config, artifacts)
|
|
31
61
|
};
|
|
32
62
|
}
|
|
33
63
|
export function buildWhisperReloadConfig(config) {
|
|
@@ -39,9 +69,15 @@ export function buildWhisperReloadConfig(config) {
|
|
|
39
69
|
})
|
|
40
70
|
};
|
|
41
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* A reload merges into the loaded config, so leaving `speakerAttribution` out
|
|
74
|
+
* keeps the diarization model the load resolved; the reload path has no
|
|
75
|
+
* artifacts to resolve a new source with.
|
|
76
|
+
*/
|
|
42
77
|
export function buildParakeetReloadConfig(config) {
|
|
78
|
+
const { speakerAttribution: _speakerAttribution, ...rest } = config;
|
|
43
79
|
return {
|
|
44
|
-
parakeetConfig: omitUndefined(
|
|
80
|
+
parakeetConfig: omitUndefined(rest)
|
|
45
81
|
};
|
|
46
82
|
}
|
|
47
83
|
export function buildMossTranscribeEngineConfig(config) {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { FitLoad, FitRequestPlan } from '../../../schemas/plugin.js';
|
|
2
|
+
/**
|
|
3
|
+
* The longest single transcribe the projection has to cover. Whisper's KV cache
|
|
4
|
+
* and decode graph are sized from it; unset leaves the engine's own default,
|
|
5
|
+
* which is the worst case it supports.
|
|
6
|
+
*/
|
|
7
|
+
export declare function audioSeconds(durationMs: number | undefined): number | undefined;
|
|
8
|
+
/**
|
|
9
|
+
* Worst-case resident decoders. Each strategy carries its own count, and `-1`
|
|
10
|
+
* is the engine's "use the default" sentinel, not a number of decoders.
|
|
11
|
+
*/
|
|
12
|
+
export declare function decoderCount(candidates: readonly (number | undefined)[]): number | undefined;
|
|
13
|
+
/** The MOSS-Transcribe fit request for a load, sized by the declared recording. */
|
|
14
|
+
export declare function mossTranscribeFitRequest(load: FitLoad): FitRequestPlan;
|
|
15
|
+
/** The Whisper fit request for a load, with its VAD model and decoder count. */
|
|
16
|
+
export declare function whisperFitRequest(load: FitLoad): FitRequestPlan;
|
|
17
|
+
/** The Parakeet fit request for a load, covering its live session when it streams. */
|
|
18
|
+
export declare function parakeetFitRequest(load: FitLoad): FitRequestPlan;
|
|
@@ -1,58 +1,59 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from '../../../model-fit/
|
|
2
|
-
export function createMossTranscribeFitRequest(params) {
|
|
3
|
-
if (params.transcriptionWorkload === undefined) {
|
|
4
|
-
return {
|
|
5
|
-
supported: false,
|
|
6
|
-
detail: 'MOSS fit requires transcriptionWorkload with explicit audioSeconds'
|
|
7
|
-
};
|
|
8
|
-
}
|
|
9
|
-
const config = (params.modelConfig ?? {});
|
|
10
|
-
const workload = params.transcriptionWorkload;
|
|
11
|
-
const request = {
|
|
12
|
-
engine: 'moss-transcribe',
|
|
13
|
-
modelPath: params.modelPath,
|
|
14
|
-
audioSeconds: workload.audioSeconds,
|
|
15
|
-
...(workload.prompt !== undefined && { prompt: workload.prompt }),
|
|
16
|
-
...(workload.hotwords !== undefined && { hotwords: workload.hotwords }),
|
|
17
|
-
...(workload.maxNewTokens !== undefined && { maxNewTokens: workload.maxNewTokens }),
|
|
18
|
-
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
19
|
-
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
20
|
-
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
|
|
21
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
22
|
-
};
|
|
23
|
-
return { supported: true, probe: { engine: 'asr-ggml', request } };
|
|
24
|
-
}
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/gpu.js';
|
|
25
2
|
/**
|
|
26
3
|
* The longest single transcribe the projection has to cover. Whisper's KV cache
|
|
27
4
|
* and decode graph are sized from it; unset leaves the engine's own default,
|
|
28
5
|
* which is the worst case it supports.
|
|
29
6
|
*/
|
|
30
|
-
function audioSeconds(durationMs) {
|
|
7
|
+
export function audioSeconds(durationMs) {
|
|
31
8
|
if (durationMs === undefined || durationMs <= 0)
|
|
32
9
|
return undefined;
|
|
33
10
|
return durationMs / 1000;
|
|
34
11
|
}
|
|
35
12
|
/**
|
|
36
|
-
* Worst-case resident decoders. Each strategy carries its own count and `-1`
|
|
37
|
-
* the engine's "use the default" sentinel
|
|
13
|
+
* Worst-case resident decoders. Each strategy carries its own count, and `-1`
|
|
14
|
+
* is the engine's "use the default" sentinel, not a number of decoders.
|
|
38
15
|
*/
|
|
39
|
-
function decoderCount(candidates) {
|
|
16
|
+
export function decoderCount(candidates) {
|
|
40
17
|
for (const candidate of candidates) {
|
|
41
18
|
if (candidate !== undefined && candidate > 0)
|
|
42
19
|
return candidate;
|
|
43
20
|
}
|
|
44
21
|
return undefined;
|
|
45
22
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
const
|
|
23
|
+
/** The MOSS-Transcribe fit request for a load, sized by the declared recording. */
|
|
24
|
+
export function mossTranscribeFitRequest(load) {
|
|
25
|
+
const workload = load.transcriptionWorkload;
|
|
26
|
+
if (workload === undefined) {
|
|
27
|
+
return { unknown: 'a MOSS projection needs a transcriptionWorkload with its audioSeconds' };
|
|
28
|
+
}
|
|
29
|
+
const config = (load.modelConfig ?? {});
|
|
30
|
+
const request = {
|
|
31
|
+
engine: 'moss-transcribe',
|
|
32
|
+
modelPath: load.modelPath,
|
|
33
|
+
audioSeconds: workload.audioSeconds,
|
|
34
|
+
...(workload.prompt !== undefined && { prompt: workload.prompt }),
|
|
35
|
+
...(workload.hotwords !== undefined && { hotwords: workload.hotwords }),
|
|
36
|
+
...(workload.maxNewTokens !== undefined && { maxNewTokens: workload.maxNewTokens }),
|
|
37
|
+
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
38
|
+
// An explicit selector projects on that device and overrides the GPU gate,
|
|
39
|
+
// as it does at load; a missing device rejects the fit.
|
|
40
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
41
|
+
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
42
|
+
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir })
|
|
43
|
+
};
|
|
44
|
+
return { request };
|
|
45
|
+
}
|
|
46
|
+
/** The Whisper fit request for a load, with its VAD model and decoder count. */
|
|
47
|
+
export function whisperFitRequest(load) {
|
|
48
|
+
const config = (load.modelConfig ?? {});
|
|
49
|
+
const vadModelPath = load.artifacts?.['vadModelPath'];
|
|
49
50
|
const decoders = decoderCount([
|
|
50
51
|
config.strategy === 'beam_search' ? config.beam_search_beam_size : config.greedy_best_of
|
|
51
52
|
]);
|
|
52
53
|
const seconds = audioSeconds(config.duration_ms);
|
|
53
54
|
const request = {
|
|
54
55
|
engine: 'whisper',
|
|
55
|
-
modelPath:
|
|
56
|
+
modelPath: load.modelPath,
|
|
56
57
|
gpuLayers: gpuLayersFromGate(config.contextParams?.use_gpu),
|
|
57
58
|
...(vadModelPath !== undefined && { vadModelPath }),
|
|
58
59
|
...(config.contextParams?.flash_attn !== undefined && {
|
|
@@ -62,35 +63,43 @@ export function createWhisperFitRequest(params) {
|
|
|
62
63
|
gpuDevice: config.contextParams.gpu_device
|
|
63
64
|
}),
|
|
64
65
|
...(decoders !== undefined && { decoders }),
|
|
65
|
-
...(seconds !== undefined && { audioSeconds: seconds })
|
|
66
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
66
|
+
...(seconds !== undefined && { audioSeconds: seconds })
|
|
67
67
|
};
|
|
68
|
-
return {
|
|
68
|
+
return { request };
|
|
69
69
|
}
|
|
70
70
|
/**
|
|
71
71
|
* The encoder's left context and Sortformer's rolling history size its live
|
|
72
72
|
* state, and the fit request measures that state in frames while the load sets
|
|
73
|
-
* it in milliseconds. Converting needs the model's frame rate,
|
|
74
|
-
*
|
|
75
|
-
* projected against the engine's default window.
|
|
73
|
+
* it in milliseconds. Converting needs the model's frame rate, a property of
|
|
74
|
+
* the checkpoint, so a streaming load that pins either is not projected.
|
|
76
75
|
*/
|
|
77
76
|
const UNREPRESENTABLE_STREAMING = [
|
|
78
77
|
'streamingLeftContextMs',
|
|
79
78
|
'streamingHistoryMs'
|
|
80
79
|
];
|
|
81
|
-
|
|
82
|
-
|
|
80
|
+
/** The Parakeet fit request for a load, covering its live session when it streams. */
|
|
81
|
+
export function parakeetFitRequest(load) {
|
|
82
|
+
const config = (load.modelConfig ?? {});
|
|
83
83
|
if (config.streaming === true) {
|
|
84
84
|
for (const key of UNREPRESENTABLE_STREAMING) {
|
|
85
|
-
if (config[key] !== undefined)
|
|
86
|
-
return {
|
|
87
|
-
}
|
|
85
|
+
if (config[key] !== undefined)
|
|
86
|
+
return { unknown: `the fitter cannot represent ${key}` };
|
|
88
87
|
}
|
|
89
88
|
}
|
|
89
|
+
// The addon projects one model, and a speaker-attributed load holds two,
|
|
90
|
+
// so a one-model verdict would understate it.
|
|
91
|
+
if (config.speakerAttribution !== undefined) {
|
|
92
|
+
return {
|
|
93
|
+
unknown: 'the fitter projects one model; assess the speaker-attribution diarization model as its own load'
|
|
94
|
+
};
|
|
95
|
+
}
|
|
90
96
|
const request = {
|
|
91
97
|
engine: 'parakeet',
|
|
92
|
-
modelPath:
|
|
98
|
+
modelPath: load.modelPath,
|
|
93
99
|
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
100
|
+
// An explicit selector projects on that device and overrides the GPU gate,
|
|
101
|
+
// as it does at load; a missing device rejects the fit.
|
|
102
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
94
103
|
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
95
104
|
// The long-form window bounds the offline encoder graph the projection
|
|
96
105
|
// sizes, so an explicit window (or the one-pass sentinel) reaches the fit.
|
|
@@ -103,32 +112,7 @@ export function createParakeetFitRequest(params) {
|
|
|
103
112
|
// Only a streaming load holds a live session, so outside one the chunk
|
|
104
113
|
// cadence describes no allocation the projection should cover.
|
|
105
114
|
...(config.streaming === true &&
|
|
106
|
-
config.streamingChunkMs !== undefined && { nemotronChunkMs: config.streamingChunkMs })
|
|
107
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
108
|
-
};
|
|
109
|
-
return { supported: true, probe: { engine: 'asr-ggml', request } };
|
|
110
|
-
}
|
|
111
|
-
export function createBciFitRequest(params) {
|
|
112
|
-
const config = (params.modelConfig ?? {});
|
|
113
|
-
const embedderPath = params.artifacts?.['embedderPath'];
|
|
114
|
-
const whisper = config.whisperConfig;
|
|
115
|
-
const decoders = decoderCount([whisper?.beam_search_beam_size, whisper?.greedy_best_of]);
|
|
116
|
-
const seconds = audioSeconds(whisper?.duration_ms);
|
|
117
|
-
return {
|
|
118
|
-
supported: true,
|
|
119
|
-
probe: {
|
|
120
|
-
engine: 'bci-whispercpp',
|
|
121
|
-
request: {
|
|
122
|
-
modelPath: params.modelPath,
|
|
123
|
-
...(embedderPath !== undefined && { embedderPath }),
|
|
124
|
-
gpuLayers: gpuLayersFromGate(config.contextParams?.use_gpu ?? true),
|
|
125
|
-
...(config.contextParams?.gpu_device !== undefined && {
|
|
126
|
-
gpuDevice: config.contextParams.gpu_device
|
|
127
|
-
}),
|
|
128
|
-
...(decoders !== undefined && { decoders }),
|
|
129
|
-
...(seconds !== undefined && { audioSeconds: seconds }),
|
|
130
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
131
|
-
}
|
|
132
|
-
}
|
|
115
|
+
config.streamingChunkMs !== undefined && { nemotronChunkMs: config.streamingChunkMs })
|
|
133
116
|
};
|
|
117
|
+
return { request };
|
|
134
118
|
}
|
|
@@ -3,6 +3,9 @@ export declare function resolveAudioGenConfig(config: Record<string, unknown>, c
|
|
|
3
3
|
config: {
|
|
4
4
|
engine: "minimax";
|
|
5
5
|
device?: "cpu" | "gpu" | "auto" | undefined;
|
|
6
|
+
coremlProfileDir?: string | undefined;
|
|
7
|
+
coremlComputeUnits?: "CPU_ONLY" | "CPU_AND_GPU" | "CPU_AND_NE" | "ALL" | undefined;
|
|
8
|
+
coremlConditionerPath?: string | undefined;
|
|
6
9
|
inferenceSteps?: number | undefined;
|
|
7
10
|
cfgScale?: number | undefined;
|
|
8
11
|
useGPU?: boolean | undefined;
|
|
@@ -18,6 +21,9 @@ export declare function resolveAudioGenConfig(config: Record<string, unknown>, c
|
|
|
18
21
|
inferenceSteps?: number | undefined;
|
|
19
22
|
shift?: number | undefined;
|
|
20
23
|
nGpuLayers?: number | undefined;
|
|
24
|
+
backend?: string | undefined;
|
|
25
|
+
lmBackend?: string | undefined;
|
|
26
|
+
dumpStagesDir?: string | undefined;
|
|
21
27
|
useGPU?: boolean | undefined;
|
|
22
28
|
threads?: number | undefined;
|
|
23
29
|
backendsDir?: string | undefined;
|
|
@@ -10,6 +10,15 @@ async function resolveMinimaxConfig(parsed, ctx) {
|
|
|
10
10
|
if (ctx.platform === 'android' || ctx.platform === 'ios') {
|
|
11
11
|
throw new ModelLoadFailedError('MiniMax-Music3 is available on desktop only');
|
|
12
12
|
}
|
|
13
|
+
// Refused before the model pair is resolved: the addon would only say so
|
|
14
|
+
// once several gigabytes had been downloaded. Only the platform is known
|
|
15
|
+
// here; the addon still checks the macOS version at load.
|
|
16
|
+
if (parsed.coremlProfileDir !== undefined && ctx.platform !== 'darwin') {
|
|
17
|
+
throw new ModelLoadFailedError('MiniMax Core ML is available on macOS only');
|
|
18
|
+
}
|
|
19
|
+
if (parsed.coremlConditionerPath !== undefined && ctx.platform !== 'darwin') {
|
|
20
|
+
throw new ModelLoadFailedError('The MiniMax Core ML conditioner is available on macOS only');
|
|
21
|
+
}
|
|
13
22
|
const { lmModelSrc, synthModelSrc, ...runtimeConfig } = parsed;
|
|
14
23
|
// Resolve sequentially for the same stall-timeout reason documented for the
|
|
15
24
|
// larger ACE-Step artifact set below.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from '../../../model-fit/
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/gpu.js';
|
|
2
2
|
/**
|
|
3
3
|
* `@qvac/audiogen-ggml` is an optional peer dependency, so its value exports
|
|
4
4
|
* are unavailable to a host that ships without it. The engine discriminator is
|
|
@@ -12,14 +12,15 @@ const MINIMAX = 'minimax';
|
|
|
12
12
|
*
|
|
13
13
|
* `device` is passed through as the load takes it. Without one, the GPU switch
|
|
14
14
|
* maps to `auto` as it does on load: the fitter reads a positive layer count
|
|
15
|
-
* that way.
|
|
16
|
-
*
|
|
15
|
+
* that way. A Core ML profile or conditioner is passed through too: the fitter
|
|
16
|
+
* cannot size either route and answers `unknown` for it. The plugin refuses
|
|
17
|
+
* MiniMax on mobile before this is reached.
|
|
17
18
|
*/
|
|
18
|
-
function minimaxFitRequest(config, artifacts
|
|
19
|
+
function minimaxFitRequest(config, artifacts) {
|
|
19
20
|
const lmPath = artifacts['lmModelPath'];
|
|
20
21
|
const synthPath = artifacts['synthModelPath'];
|
|
21
22
|
if (lmPath === undefined || synthPath === undefined) {
|
|
22
|
-
return {
|
|
23
|
+
return { unknown: 'the minimax model pair is not resolved' };
|
|
23
24
|
}
|
|
24
25
|
const request = {
|
|
25
26
|
engine: MINIMAX,
|
|
@@ -28,17 +29,23 @@ function minimaxFitRequest(config, artifacts, marginBytes) {
|
|
|
28
29
|
...(config.device !== undefined
|
|
29
30
|
? { device: config.device }
|
|
30
31
|
: { gpuLayers: gpuLayersFromGate(config.useGPU) }),
|
|
31
|
-
...(config.
|
|
32
|
-
|
|
32
|
+
...(config.coremlConditionerPath !== undefined && {
|
|
33
|
+
coremlConditionerPath: config.coremlConditionerPath
|
|
34
|
+
}),
|
|
35
|
+
...(config.coremlProfileDir !== undefined && { coremlProfileDir: config.coremlProfileDir }),
|
|
36
|
+
...(config.coremlComputeUnits !== undefined && {
|
|
37
|
+
coremlComputeUnits: config.coremlComputeUnits
|
|
38
|
+
}),
|
|
39
|
+
...(config.threads !== undefined && { threads: config.threads })
|
|
33
40
|
};
|
|
34
|
-
return {
|
|
41
|
+
return { request };
|
|
35
42
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
43
|
+
/** The audiogen fit request for a load, from its resolved stage checkpoints. */
|
|
44
|
+
export function audiogenFitRequest(load) {
|
|
45
|
+
const config = (load.modelConfig ?? {});
|
|
46
|
+
const artifacts = load.artifacts ?? {};
|
|
47
|
+
if (config.engine === MINIMAX)
|
|
48
|
+
return minimaxFitRequest(config, artifacts);
|
|
42
49
|
const textEncoderPath = artifacts['textEncModelPath'];
|
|
43
50
|
const lmPath = artifacts['lmModelPath'];
|
|
44
51
|
const ditPath = artifacts['ditModelPath'];
|
|
@@ -47,17 +54,20 @@ export function createAudiogenFitRequest(params) {
|
|
|
47
54
|
lmPath === undefined ||
|
|
48
55
|
ditPath === undefined ||
|
|
49
56
|
vaePath === undefined) {
|
|
50
|
-
return {
|
|
57
|
+
return { unknown: 'the ace-step stage checkpoints are not all resolved' };
|
|
51
58
|
}
|
|
52
59
|
const layers = 'nGpuLayers' in config ? config.nGpuLayers : undefined;
|
|
60
|
+
// A named backend moves a stage to its own device, so the fitter is given
|
|
61
|
+
// the placement the load will ask for and sizes each device it names.
|
|
53
62
|
const request = {
|
|
54
63
|
textEncoderPath,
|
|
55
64
|
lmPath,
|
|
56
65
|
ditPath,
|
|
57
66
|
vaePath,
|
|
58
67
|
gpuLayers: gpuLayersFromGate(config.useGPU, layers),
|
|
59
|
-
...(config.
|
|
60
|
-
...(
|
|
68
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
69
|
+
...(config.lmBackend !== undefined && { lmBackend: config.lmBackend }),
|
|
70
|
+
...(config.threads !== undefined && { threads: config.threads })
|
|
61
71
|
};
|
|
62
|
-
return {
|
|
72
|
+
return { request };
|
|
63
73
|
}
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
+
import type Buffer from 'bare-buffer';
|
|
2
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
3
|
import { type AudioEditStreamRequest, type AudioEditStreamResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
-
export declare function audioEditStream(request: AudioEditStreamRequest): AsyncGenerator<AudioEditStreamResponse>;
|
|
4
|
+
export declare function audioEditStream(request: AudioEditStreamRequest, _inputStream?: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<AudioEditStreamResponse>;
|
|
@@ -9,10 +9,11 @@ const REPAINT_MODES = {
|
|
|
9
9
|
balanced: RepaintMode.Balanced,
|
|
10
10
|
aggressive: RepaintMode.Aggressive
|
|
11
11
|
};
|
|
12
|
-
export function audioEditStream(request) {
|
|
12
|
+
export function audioEditStream(request, _inputStream, context) {
|
|
13
13
|
return streamAudioGenRun({
|
|
14
14
|
type: 'audioEditStream',
|
|
15
15
|
request,
|
|
16
|
+
context,
|
|
16
17
|
async start(model, ctx, logger) {
|
|
17
18
|
// The source and reference are decoded before the run is admitted so
|
|
18
19
|
// the model slot is never held by a request that fails on input
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { AudioGenOutputFormat, AudioGenStreamResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
+
export interface PcmLayout {
|
|
3
|
+
sampleRate: number;
|
|
4
|
+
channels: number;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Proves the host can encode `format` before a generation is admitted: the
|
|
8
|
+
* FFmpeg build is the addon's, and a missing encoder would otherwise only
|
|
9
|
+
* surface after the run had finished. `wav` is written in JS and needs no
|
|
10
|
+
* probe. The outcome is remembered per format, so only the first request for
|
|
11
|
+
* a format pays for the probe.
|
|
12
|
+
*/
|
|
13
|
+
export declare function assertOutputFormatAvailable(format: AudioGenOutputFormat): void;
|
|
14
|
+
/** Forgets every probe outcome. For tests that swap the encoder. */
|
|
15
|
+
export declare function resetOutputFormatProbes(): void;
|
|
16
|
+
/** Encodes a run's interleaved Int16 PCM into the frame field that carries it. */
|
|
17
|
+
export declare function encodeRunAudio(chunks: readonly Uint8Array[], format: AudioGenOutputFormat, layout: PcmLayout): NonNullable<AudioGenStreamResponse['encoded']>;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { AudioGen } from '@qvac/audiogen-ggml';
|
|
2
|
+
import { AudioEncodeFailedError } from '../../../../errors/index.js';
|
|
3
|
+
// A hundredth of a second of stereo silence at 48 kHz: enough for every
|
|
4
|
+
// encoder to open, emit a frame and close its container.
|
|
5
|
+
const PROBE_SAMPLE_RATE = 48000;
|
|
6
|
+
const PROBE_CHANNELS = 2;
|
|
7
|
+
const PROBE_FRAMES = 480;
|
|
8
|
+
function encode(pcm, format, layout) {
|
|
9
|
+
try {
|
|
10
|
+
return AudioGen.encode(pcm, format, layout);
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
throw new AudioEncodeFailedError(format, error instanceof Error ? error.message : String(error), error);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
// The addon's FFmpeg build is fixed for the life of the process, so each
|
|
17
|
+
// format is probed once: either the encoder is there, or the entry keeps what
|
|
18
|
+
// the failed probe reported so the same error can be raised again.
|
|
19
|
+
const probeResults = new Map();
|
|
20
|
+
/**
|
|
21
|
+
* Proves the host can encode `format` before a generation is admitted: the
|
|
22
|
+
* FFmpeg build is the addon's, and a missing encoder would otherwise only
|
|
23
|
+
* surface after the run had finished. `wav` is written in JS and needs no
|
|
24
|
+
* probe. The outcome is remembered per format, so only the first request for
|
|
25
|
+
* a format pays for the probe.
|
|
26
|
+
*/
|
|
27
|
+
export function assertOutputFormatAvailable(format) {
|
|
28
|
+
if (format === 'wav')
|
|
29
|
+
return;
|
|
30
|
+
const known = probeResults.get(format);
|
|
31
|
+
if (known === 'available')
|
|
32
|
+
return;
|
|
33
|
+
if (known !== undefined)
|
|
34
|
+
throw new AudioEncodeFailedError(format, known.details);
|
|
35
|
+
try {
|
|
36
|
+
encode(new Uint8Array(PROBE_FRAMES * PROBE_CHANNELS * Int16Array.BYTES_PER_ELEMENT), format, {
|
|
37
|
+
sampleRate: PROBE_SAMPLE_RATE,
|
|
38
|
+
channels: PROBE_CHANNELS
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
probeResults.set(format, {
|
|
43
|
+
details: error instanceof AudioEncodeFailedError ? error.details : String(error)
|
|
44
|
+
});
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
probeResults.set(format, 'available');
|
|
48
|
+
}
|
|
49
|
+
/** Forgets every probe outcome. For tests that swap the encoder. */
|
|
50
|
+
export function resetOutputFormatProbes() {
|
|
51
|
+
probeResults.clear();
|
|
52
|
+
}
|
|
53
|
+
/** Encodes a run's interleaved Int16 PCM into the frame field that carries it. */
|
|
54
|
+
export function encodeRunAudio(chunks, format, layout) {
|
|
55
|
+
const pcm = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.byteLength, 0));
|
|
56
|
+
let offset = 0;
|
|
57
|
+
for (const chunk of chunks) {
|
|
58
|
+
pcm.set(chunk, offset);
|
|
59
|
+
offset += chunk.byteLength;
|
|
60
|
+
}
|
|
61
|
+
const encoded = encode(pcm, format, layout);
|
|
62
|
+
return {
|
|
63
|
+
data: Buffer.from(encoded.data.buffer, encoded.data.byteOffset, encoded.data.byteLength).toString('base64'),
|
|
64
|
+
format,
|
|
65
|
+
mimeType: encoded.mimeType,
|
|
66
|
+
extension: encoded.extension
|
|
67
|
+
};
|
|
68
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
2
|
import { AudioGen } from '@qvac/audiogen-ggml';
|
|
2
|
-
import { type AudioGenStreamResponse, type AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
|
|
3
|
+
import { type AudioGenOutputFormat, type AudioGenStreamResponse, type AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
|
|
3
4
|
import { type Logger } from '../../../../logging/index.js';
|
|
4
5
|
import { type RequestContext } from '../../../../runtime/index.js';
|
|
5
6
|
/**
|
|
@@ -26,7 +27,12 @@ export interface AudioGenRunOptions<TType extends AudioGenRunType> {
|
|
|
26
27
|
request: {
|
|
27
28
|
modelId: string;
|
|
28
29
|
requestId?: string | undefined;
|
|
30
|
+
sessionId?: string | undefined;
|
|
31
|
+
/** Generation and editing only: also deliver the run in this container. */
|
|
32
|
+
outputFormat?: AudioGenOutputFormat | undefined;
|
|
29
33
|
};
|
|
34
|
+
/** The invoke this run belongs to, when reached through `pluginInvoke`. */
|
|
35
|
+
context?: PluginHandlerContext | undefined;
|
|
30
36
|
/**
|
|
31
37
|
* Prepares the inputs and admits the native job once the request holds the
|
|
32
38
|
* model slot. Preparation (decoding audio) can take real time, so the
|
|
@@ -39,8 +45,10 @@ export interface AudioGenRunOptions<TType extends AudioGenRunType> {
|
|
|
39
45
|
}
|
|
40
46
|
/**
|
|
41
47
|
* Runs one AudioGen job under the request registry and streams it as
|
|
42
|
-
* `TType` frames: progress ticks, PCM chunks,
|
|
43
|
-
*
|
|
48
|
+
* `TType` frames: progress ticks, PCM chunks, the encoded file (or the reason
|
|
49
|
+
* it could not be encoded) when the request named an `outputFormat`, then the
|
|
50
|
+
* terminal frame with stats and
|
|
51
|
+
* backend diagnostics. Owns admission (`kind: 'audiogen'`, one
|
|
44
52
|
* active job per model), abort-to-cancel wiring, and the cancelled terminal.
|
|
45
53
|
*/
|
|
46
54
|
export declare function streamAudioGenRun<TType extends AudioGenRunType>(options: AudioGenRunOptions<TType>): AsyncGenerator<AudioGenRunFrame<TType>>;
|