@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -50,8 +50,8 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../../../schemas/plugin.js';
|
|
53
54
|
import { AudioGen, audiogenBackendName, audiogenGpuFallbackReason } from '@qvac/audiogen-ggml';
|
|
54
|
-
import { z } from 'zod';
|
|
55
55
|
import { audioGenStatsSchema } from '../../../../schemas/audio-gen.js';
|
|
56
56
|
import { graphicsDriverSchema } from '../../../../schemas/index.js';
|
|
57
57
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
@@ -59,19 +59,28 @@ import { attachBackendDiagnostics } from '../../../../profiling/backend-diagnost
|
|
|
59
59
|
import { getModel } from '../../../../runtime/model-registry.js';
|
|
60
60
|
import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
|
|
61
61
|
import { generateRequestId } from '../../../../runtime/request-id.js';
|
|
62
|
-
import { ModelOperationNotSupportedError } from '../../../../errors/index.js';
|
|
62
|
+
import { AudioEncodeFailedError, ModelOperationNotSupportedError } from '../../../../errors/index.js';
|
|
63
|
+
import { assertOutputFormatAvailable, encodeRunAudio } from '../../../../plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js';
|
|
63
64
|
/**
|
|
64
65
|
* Runs one AudioGen job under the request registry and streams it as
|
|
65
|
-
* `TType` frames: progress ticks, PCM chunks,
|
|
66
|
-
*
|
|
66
|
+
* `TType` frames: progress ticks, PCM chunks, the encoded file (or the reason
|
|
67
|
+
* it could not be encoded) when the request named an `outputFormat`, then the
|
|
68
|
+
* terminal frame with stats and
|
|
69
|
+
* backend diagnostics. Owns admission (`kind: 'audiogen'`, one
|
|
67
70
|
* active job per model), abort-to-cancel wiring, and the cancelled terminal.
|
|
68
71
|
*/
|
|
69
72
|
export async function* streamAudioGenRun(options) {
|
|
70
73
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
71
74
|
try {
|
|
72
|
-
const { type, request } = options;
|
|
75
|
+
const { type, request, context } = options;
|
|
76
|
+
// Checked before the request queues for the model: an encoder the host lacks
|
|
77
|
+
// must not cost a whole generation to find out about.
|
|
78
|
+
if (request.outputFormat !== undefined)
|
|
79
|
+
assertOutputFormatAvailable(request.outputFormat);
|
|
73
80
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
74
81
|
requestId: request.requestId ?? generateRequestId(),
|
|
82
|
+
sessionId: request.sessionId,
|
|
83
|
+
...requestParent(context),
|
|
75
84
|
kind: 'audiogen',
|
|
76
85
|
modelId: request.modelId
|
|
77
86
|
}), true);
|
|
@@ -103,6 +112,9 @@ export async function* streamAudioGenRun(options) {
|
|
|
103
112
|
});
|
|
104
113
|
let response;
|
|
105
114
|
let streamedUnderstand;
|
|
115
|
+
// Kept only when the run is also to be encoded.
|
|
116
|
+
const encodeChunks = [];
|
|
117
|
+
let encodeLayout;
|
|
106
118
|
try {
|
|
107
119
|
response = await options.start(model, ctx, logger);
|
|
108
120
|
if (response !== undefined) {
|
|
@@ -129,6 +141,12 @@ export async function* streamAudioGenRun(options) {
|
|
|
129
141
|
if (!('outputArray' in chunk))
|
|
130
142
|
continue;
|
|
131
143
|
const pcm = new Uint8Array(chunk.outputArray.buffer, chunk.outputArray.byteOffset, chunk.outputArray.byteLength);
|
|
144
|
+
if (request.outputFormat !== undefined) {
|
|
145
|
+
// Copied: the view is over the addon's buffer, which is held here
|
|
146
|
+
// until the run ends and must not change under a later chunk.
|
|
147
|
+
encodeChunks.push(pcm.slice());
|
|
148
|
+
encodeLayout = { sampleRate: chunk.sampleRate, channels: chunk.channels };
|
|
149
|
+
}
|
|
132
150
|
yield {
|
|
133
151
|
type,
|
|
134
152
|
data: Buffer.from(pcm).toString('base64'),
|
|
@@ -148,6 +166,15 @@ export async function* streamAudioGenRun(options) {
|
|
|
148
166
|
yield { type, done: true, stopReason: 'cancelled' };
|
|
149
167
|
return;
|
|
150
168
|
}
|
|
169
|
+
if (request.outputFormat !== undefined && encodeLayout !== undefined) {
|
|
170
|
+
// The PCM is already with the caller. A failure here costs only the file:
|
|
171
|
+
// it is reported on its own frame and the run still ends as completed.
|
|
172
|
+
yield {
|
|
173
|
+
type,
|
|
174
|
+
...encodeOutcome(encodeChunks, request.outputFormat, encodeLayout, logger),
|
|
175
|
+
done: false
|
|
176
|
+
};
|
|
177
|
+
}
|
|
151
178
|
const raw = await response.await();
|
|
152
179
|
// `understand.audioCodes` arrives as an Int32Array, which the wire schema —
|
|
153
180
|
// and anything downstream of JSON — needs as a plain array of integers.
|
|
@@ -157,7 +184,7 @@ export async function* streamAudioGenRun(options) {
|
|
|
157
184
|
? { ...raw, understand: streamedUnderstand ?? toUnderstandResult(raw.understand) }
|
|
158
185
|
: raw;
|
|
159
186
|
const stats = audioGenStatsSchema.parse(normalized);
|
|
160
|
-
const diagnostics = buildBackendDiagnostics(
|
|
187
|
+
const diagnostics = buildBackendDiagnostics(stats);
|
|
161
188
|
const terminal = {
|
|
162
189
|
type,
|
|
163
190
|
done: true,
|
|
@@ -177,11 +204,20 @@ export async function* streamAudioGenRun(options) {
|
|
|
177
204
|
await result_1;
|
|
178
205
|
}
|
|
179
206
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
207
|
+
function encodeOutcome(chunks, format, layout, logger) {
|
|
208
|
+
try {
|
|
209
|
+
return { encoded: encodeRunAudio(chunks, format, layout) };
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
const details = error instanceof AudioEncodeFailedError
|
|
213
|
+
? error.details
|
|
214
|
+
: error instanceof Error
|
|
215
|
+
? error.message
|
|
216
|
+
: String(error);
|
|
217
|
+
logger.warn(`[audiogen] encoding the finished run as ${format} failed; delivering PCM only: ${details ?? 'no detail'}`);
|
|
218
|
+
return { encodeError: { format, ...(details !== undefined && { details }) } };
|
|
219
|
+
}
|
|
220
|
+
}
|
|
185
221
|
/**
|
|
186
222
|
* Normalizes the addon's understand result for the wire: `audioCodes` arrives
|
|
187
223
|
* as an `Int32Array`, which neither the schema nor a JSON transport accepts.
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
+
import type Buffer from 'bare-buffer';
|
|
2
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
3
|
import type { AudioGenStreamRequest, AudioGenStreamResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
-
export declare function audioGenStream(request: AudioGenStreamRequest): AsyncGenerator<AudioGenStreamResponse>;
|
|
4
|
+
export declare function audioGenStream(request: AudioGenStreamRequest, _inputStream?: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<AudioGenStreamResponse>;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { resolveAudioGenInputs } from '../../../../plugins/builtin/audiogen-ggml/ops/audio-gen-input.js';
|
|
2
2
|
import { streamAudioGenRun } from '../../../../plugins/builtin/audiogen-ggml/ops/audio-gen-run.js';
|
|
3
|
-
export function audioGenStream(request) {
|
|
3
|
+
export function audioGenStream(request, _inputStream, context) {
|
|
4
4
|
return streamAudioGenRun({
|
|
5
5
|
type: 'audioGenStream',
|
|
6
6
|
request,
|
|
7
|
+
context,
|
|
7
8
|
async start(model, ctx, logger) {
|
|
8
9
|
// Reference/source audio is decoded before the run is admitted so the
|
|
9
10
|
// model slot is never held by a request that fails on input decoding.
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type Buffer from 'bare-buffer';
|
|
2
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
3
|
import type { AudioUnderstandRequest, AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
|
|
2
4
|
/**
|
|
3
5
|
* ACE-Step's reverse pipeline: encode a recording, recover its FSQ semantic
|
|
@@ -5,4 +7,4 @@ import type { AudioUnderstandRequest, AudioUnderstandResponse } from '../../../.
|
|
|
5
7
|
* cancellation, and streams the description as one `understand` frame before
|
|
6
8
|
* the terminal frame repeats it on `stats`.
|
|
7
9
|
*/
|
|
8
|
-
export declare function audioUnderstand(request: AudioUnderstandRequest): AsyncGenerator<AudioUnderstandResponse>;
|
|
10
|
+
export declare function audioUnderstand(request: AudioUnderstandRequest, _inputStream?: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<AudioUnderstandResponse>;
|
|
@@ -6,10 +6,11 @@ import { streamAudioGenRun } from '../../../../plugins/builtin/audiogen-ggml/ops
|
|
|
6
6
|
* cancellation, and streams the description as one `understand` frame before
|
|
7
7
|
* the terminal frame repeats it on `stats`.
|
|
8
8
|
*/
|
|
9
|
-
export function audioUnderstand(request) {
|
|
9
|
+
export function audioUnderstand(request, _inputStream, context) {
|
|
10
10
|
return streamAudioGenRun({
|
|
11
11
|
type: 'audioUnderstand',
|
|
12
12
|
request,
|
|
13
|
+
context,
|
|
13
14
|
async start(model, ctx) {
|
|
14
15
|
// Decoded before the job is admitted, so the model slot is never held by
|
|
15
16
|
// a request that fails on input decoding.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
|
|
2
2
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
3
3
|
import { resolveAudioGenConfig } from '../../../plugins/builtin/audiogen-ggml/config.js';
|
|
4
|
+
import { audiogenFitRequest } from '../../../plugins/builtin/audiogen-ggml/fit.js';
|
|
4
5
|
export declare const audioGenPlugin: {
|
|
5
6
|
modelType: "audiogen-ggml";
|
|
6
7
|
displayName: string;
|
|
@@ -9,6 +10,9 @@ export declare const audioGenPlugin: {
|
|
|
9
10
|
inferenceSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
10
11
|
shift: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
11
12
|
nGpuLayers: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
13
|
+
backend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
14
|
+
lmBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
15
|
+
dumpStagesDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
12
16
|
useGPU: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
13
17
|
threads: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
14
18
|
backendsDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -30,6 +34,7 @@ export declare const audioGenPlugin: {
|
|
|
30
34
|
audiogen: "audiogen";
|
|
31
35
|
tts: "tts";
|
|
32
36
|
ocr: "ocr";
|
|
37
|
+
classification: "classification";
|
|
33
38
|
vla: "vla";
|
|
34
39
|
"llamacpp-completion": "llamacpp-completion";
|
|
35
40
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
@@ -46,13 +51,13 @@ export declare const audioGenPlugin: {
|
|
|
46
51
|
"sdcpp-generation": "sdcpp-generation";
|
|
47
52
|
"audiogen-ggml": "audiogen-ggml";
|
|
48
53
|
"ggml-vla": "ggml-vla";
|
|
54
|
+
"ggml-image-embedding": "ggml-image-embedding";
|
|
49
55
|
"ggml-classification": "ggml-classification";
|
|
50
56
|
llm: "llm";
|
|
51
57
|
whisper: "whisper";
|
|
52
58
|
bci: "bci";
|
|
53
59
|
nmt: "nmt";
|
|
54
60
|
parakeet: "parakeet";
|
|
55
|
-
classification: "classification";
|
|
56
61
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
57
62
|
}, import("zod/v4/core").$strip>]>;
|
|
58
63
|
lmModelSrc: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -72,6 +77,7 @@ export declare const audioGenPlugin: {
|
|
|
72
77
|
audiogen: "audiogen";
|
|
73
78
|
tts: "tts";
|
|
74
79
|
ocr: "ocr";
|
|
80
|
+
classification: "classification";
|
|
75
81
|
vla: "vla";
|
|
76
82
|
"llamacpp-completion": "llamacpp-completion";
|
|
77
83
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
@@ -88,13 +94,13 @@ export declare const audioGenPlugin: {
|
|
|
88
94
|
"sdcpp-generation": "sdcpp-generation";
|
|
89
95
|
"audiogen-ggml": "audiogen-ggml";
|
|
90
96
|
"ggml-vla": "ggml-vla";
|
|
97
|
+
"ggml-image-embedding": "ggml-image-embedding";
|
|
91
98
|
"ggml-classification": "ggml-classification";
|
|
92
99
|
llm: "llm";
|
|
93
100
|
whisper: "whisper";
|
|
94
101
|
bci: "bci";
|
|
95
102
|
nmt: "nmt";
|
|
96
103
|
parakeet: "parakeet";
|
|
97
|
-
classification: "classification";
|
|
98
104
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
99
105
|
}, import("zod/v4/core").$strip>]>;
|
|
100
106
|
ditModelSrc: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -114,6 +120,7 @@ export declare const audioGenPlugin: {
|
|
|
114
120
|
audiogen: "audiogen";
|
|
115
121
|
tts: "tts";
|
|
116
122
|
ocr: "ocr";
|
|
123
|
+
classification: "classification";
|
|
117
124
|
vla: "vla";
|
|
118
125
|
"llamacpp-completion": "llamacpp-completion";
|
|
119
126
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
@@ -130,13 +137,13 @@ export declare const audioGenPlugin: {
|
|
|
130
137
|
"sdcpp-generation": "sdcpp-generation";
|
|
131
138
|
"audiogen-ggml": "audiogen-ggml";
|
|
132
139
|
"ggml-vla": "ggml-vla";
|
|
140
|
+
"ggml-image-embedding": "ggml-image-embedding";
|
|
133
141
|
"ggml-classification": "ggml-classification";
|
|
134
142
|
llm: "llm";
|
|
135
143
|
whisper: "whisper";
|
|
136
144
|
bci: "bci";
|
|
137
145
|
nmt: "nmt";
|
|
138
146
|
parakeet: "parakeet";
|
|
139
|
-
classification: "classification";
|
|
140
147
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
141
148
|
}, import("zod/v4/core").$strip>]>;
|
|
142
149
|
vaeModelSrc: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -156,6 +163,7 @@ export declare const audioGenPlugin: {
|
|
|
156
163
|
audiogen: "audiogen";
|
|
157
164
|
tts: "tts";
|
|
158
165
|
ocr: "ocr";
|
|
166
|
+
classification: "classification";
|
|
159
167
|
vla: "vla";
|
|
160
168
|
"llamacpp-completion": "llamacpp-completion";
|
|
161
169
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
@@ -172,13 +180,13 @@ export declare const audioGenPlugin: {
|
|
|
172
180
|
"sdcpp-generation": "sdcpp-generation";
|
|
173
181
|
"audiogen-ggml": "audiogen-ggml";
|
|
174
182
|
"ggml-vla": "ggml-vla";
|
|
183
|
+
"ggml-image-embedding": "ggml-image-embedding";
|
|
175
184
|
"ggml-classification": "ggml-classification";
|
|
176
185
|
llm: "llm";
|
|
177
186
|
whisper: "whisper";
|
|
178
187
|
bci: "bci";
|
|
179
188
|
nmt: "nmt";
|
|
180
189
|
parakeet: "parakeet";
|
|
181
|
-
classification: "classification";
|
|
182
190
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
183
191
|
}, import("zod/v4/core").$strip>]>;
|
|
184
192
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
@@ -199,6 +207,7 @@ export declare const audioGenPlugin: {
|
|
|
199
207
|
audiogen: "audiogen";
|
|
200
208
|
tts: "tts";
|
|
201
209
|
ocr: "ocr";
|
|
210
|
+
classification: "classification";
|
|
202
211
|
vla: "vla";
|
|
203
212
|
"llamacpp-completion": "llamacpp-completion";
|
|
204
213
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
@@ -215,13 +224,13 @@ export declare const audioGenPlugin: {
|
|
|
215
224
|
"sdcpp-generation": "sdcpp-generation";
|
|
216
225
|
"audiogen-ggml": "audiogen-ggml";
|
|
217
226
|
"ggml-vla": "ggml-vla";
|
|
227
|
+
"ggml-image-embedding": "ggml-image-embedding";
|
|
218
228
|
"ggml-classification": "ggml-classification";
|
|
219
229
|
llm: "llm";
|
|
220
230
|
whisper: "whisper";
|
|
221
231
|
bci: "bci";
|
|
222
232
|
nmt: "nmt";
|
|
223
233
|
parakeet: "parakeet";
|
|
224
|
-
classification: "classification";
|
|
225
234
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
226
235
|
}, import("zod/v4/core").$strip>]>;
|
|
227
236
|
synthModelSrc: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -241,6 +250,7 @@ export declare const audioGenPlugin: {
|
|
|
241
250
|
audiogen: "audiogen";
|
|
242
251
|
tts: "tts";
|
|
243
252
|
ocr: "ocr";
|
|
253
|
+
classification: "classification";
|
|
244
254
|
vla: "vla";
|
|
245
255
|
"llamacpp-completion": "llamacpp-completion";
|
|
246
256
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
@@ -257,13 +267,13 @@ export declare const audioGenPlugin: {
|
|
|
257
267
|
"sdcpp-generation": "sdcpp-generation";
|
|
258
268
|
"audiogen-ggml": "audiogen-ggml";
|
|
259
269
|
"ggml-vla": "ggml-vla";
|
|
270
|
+
"ggml-image-embedding": "ggml-image-embedding";
|
|
260
271
|
"ggml-classification": "ggml-classification";
|
|
261
272
|
llm: "llm";
|
|
262
273
|
whisper: "whisper";
|
|
263
274
|
bci: "bci";
|
|
264
275
|
nmt: "nmt";
|
|
265
276
|
parakeet: "parakeet";
|
|
266
|
-
classification: "classification";
|
|
267
277
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
268
278
|
}, import("zod/v4/core").$strip>]>;
|
|
269
279
|
device: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
@@ -271,6 +281,14 @@ export declare const audioGenPlugin: {
|
|
|
271
281
|
gpu: "gpu";
|
|
272
282
|
auto: "auto";
|
|
273
283
|
}>>;
|
|
284
|
+
coremlProfileDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
285
|
+
coremlComputeUnits: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
286
|
+
CPU_ONLY: "CPU_ONLY";
|
|
287
|
+
CPU_AND_GPU: "CPU_AND_GPU";
|
|
288
|
+
CPU_AND_NE: "CPU_AND_NE";
|
|
289
|
+
ALL: "ALL";
|
|
290
|
+
}>>;
|
|
291
|
+
coremlConditionerPath: import("zod").ZodOptional<import("zod").ZodString>;
|
|
274
292
|
inferenceSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
275
293
|
cfgScale: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
276
294
|
useGPU: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
@@ -278,6 +296,7 @@ export declare const audioGenPlugin: {
|
|
|
278
296
|
backendsDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
279
297
|
engine: import("zod").ZodLiteral<"minimax">;
|
|
280
298
|
}, import("zod/v4/core").$strict>], "engine">;
|
|
299
|
+
fitRequest: typeof audiogenFitRequest;
|
|
281
300
|
assessFit: typeof audiogenAssessFit;
|
|
282
301
|
skipPrimaryModelPathValidation: true;
|
|
283
302
|
resolveConfig: typeof resolveAudioGenConfig;
|
|
@@ -285,7 +304,6 @@ export declare const audioGenPlugin: {
|
|
|
285
304
|
handlers: {
|
|
286
305
|
audioGenStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
287
306
|
type: import("zod").ZodLiteral<"audioGenStream">;
|
|
288
|
-
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
289
307
|
audioCodes: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodNumber>>;
|
|
290
308
|
referenceAudio: import("zod").ZodOptional<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
291
309
|
type: import("zod").ZodLiteral<"base64">;
|
|
@@ -350,6 +368,22 @@ export declare const audioGenPlugin: {
|
|
|
350
368
|
}>>;
|
|
351
369
|
audioCoverStrength: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
352
370
|
coverNoiseStrength: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
371
|
+
outputFormat: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
372
|
+
wav: "wav";
|
|
373
|
+
flac: "flac";
|
|
374
|
+
alac: "alac";
|
|
375
|
+
aiff: "aiff";
|
|
376
|
+
caf: "caf";
|
|
377
|
+
m4a: "m4a";
|
|
378
|
+
aac: "aac";
|
|
379
|
+
opus: "opus";
|
|
380
|
+
ogg: "ogg";
|
|
381
|
+
ac3: "ac3";
|
|
382
|
+
wma: "wma";
|
|
383
|
+
mp2: "mp2";
|
|
384
|
+
}>>;
|
|
385
|
+
sessionId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
386
|
+
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
353
387
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
354
388
|
progress: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
355
389
|
stage: import("zod").ZodString;
|
|
@@ -360,6 +394,42 @@ export declare const audioGenPlugin: {
|
|
|
360
394
|
sampleRate: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
361
395
|
channels: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
362
396
|
bitsPerSample: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
397
|
+
encoded: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
398
|
+
data: import("zod").ZodString;
|
|
399
|
+
format: import("zod").ZodEnum<{
|
|
400
|
+
wav: "wav";
|
|
401
|
+
flac: "flac";
|
|
402
|
+
alac: "alac";
|
|
403
|
+
aiff: "aiff";
|
|
404
|
+
caf: "caf";
|
|
405
|
+
m4a: "m4a";
|
|
406
|
+
aac: "aac";
|
|
407
|
+
opus: "opus";
|
|
408
|
+
ogg: "ogg";
|
|
409
|
+
ac3: "ac3";
|
|
410
|
+
wma: "wma";
|
|
411
|
+
mp2: "mp2";
|
|
412
|
+
}>;
|
|
413
|
+
mimeType: import("zod").ZodString;
|
|
414
|
+
extension: import("zod").ZodString;
|
|
415
|
+
}, import("zod/v4/core").$strict>>;
|
|
416
|
+
encodeError: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
417
|
+
format: import("zod").ZodEnum<{
|
|
418
|
+
wav: "wav";
|
|
419
|
+
flac: "flac";
|
|
420
|
+
alac: "alac";
|
|
421
|
+
aiff: "aiff";
|
|
422
|
+
caf: "caf";
|
|
423
|
+
m4a: "m4a";
|
|
424
|
+
aac: "aac";
|
|
425
|
+
opus: "opus";
|
|
426
|
+
ogg: "ogg";
|
|
427
|
+
ac3: "ac3";
|
|
428
|
+
wma: "wma";
|
|
429
|
+
mp2: "mp2";
|
|
430
|
+
}>;
|
|
431
|
+
details: import("zod").ZodOptional<import("zod").ZodString>;
|
|
432
|
+
}, import("zod/v4/core").$strict>>;
|
|
363
433
|
done: import("zod").ZodDefault<import("zod").ZodBoolean>;
|
|
364
434
|
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
365
435
|
cancelled: "cancelled";
|
|
@@ -371,6 +441,7 @@ export declare const audioGenPlugin: {
|
|
|
371
441
|
realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
372
442
|
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
373
443
|
backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
444
|
+
gpuFallbackReason: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
374
445
|
emittedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
375
446
|
arMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
376
447
|
conditionMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -408,16 +479,16 @@ export declare const audioGenPlugin: {
|
|
|
408
479
|
gpu: "gpu";
|
|
409
480
|
}>;
|
|
410
481
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
411
|
-
cuda: "cuda";
|
|
412
482
|
vulkan: "vulkan";
|
|
413
|
-
metal: "metal";
|
|
414
483
|
opencl: "opencl";
|
|
415
|
-
rocm: "rocm";
|
|
416
484
|
opengl: "opengl";
|
|
417
485
|
webgpu: "webgpu";
|
|
486
|
+
metal: "metal";
|
|
418
487
|
direct3d11: "direct3d11";
|
|
419
488
|
direct3d12: "direct3d12";
|
|
489
|
+
cuda: "cuda";
|
|
420
490
|
levelZero: "levelZero";
|
|
491
|
+
rocm: "rocm";
|
|
421
492
|
}>>;
|
|
422
493
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
423
494
|
name: import("zod").ZodString;
|
|
@@ -460,7 +531,6 @@ export declare const audioGenPlugin: {
|
|
|
460
531
|
value: import("zod").ZodString;
|
|
461
532
|
}, import("zod/v4/core").$strip>], "type">>;
|
|
462
533
|
type: import("zod").ZodLiteral<"audioEditStream">;
|
|
463
|
-
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
464
534
|
modelId: import("zod").ZodString;
|
|
465
535
|
operations: import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
466
536
|
type: import("zod").ZodLiteral<"flow-edit">;
|
|
@@ -499,6 +569,22 @@ export declare const audioGenPlugin: {
|
|
|
499
569
|
dcwHighScaler: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
500
570
|
inferenceSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
501
571
|
shift: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
572
|
+
outputFormat: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
573
|
+
wav: "wav";
|
|
574
|
+
flac: "flac";
|
|
575
|
+
alac: "alac";
|
|
576
|
+
aiff: "aiff";
|
|
577
|
+
caf: "caf";
|
|
578
|
+
m4a: "m4a";
|
|
579
|
+
aac: "aac";
|
|
580
|
+
opus: "opus";
|
|
581
|
+
ogg: "ogg";
|
|
582
|
+
ac3: "ac3";
|
|
583
|
+
wma: "wma";
|
|
584
|
+
mp2: "mp2";
|
|
585
|
+
}>>;
|
|
586
|
+
sessionId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
587
|
+
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
502
588
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
503
589
|
progress: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
504
590
|
stage: import("zod").ZodString;
|
|
@@ -509,6 +595,42 @@ export declare const audioGenPlugin: {
|
|
|
509
595
|
sampleRate: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
510
596
|
channels: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
511
597
|
bitsPerSample: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
598
|
+
encoded: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
599
|
+
data: import("zod").ZodString;
|
|
600
|
+
format: import("zod").ZodEnum<{
|
|
601
|
+
wav: "wav";
|
|
602
|
+
flac: "flac";
|
|
603
|
+
alac: "alac";
|
|
604
|
+
aiff: "aiff";
|
|
605
|
+
caf: "caf";
|
|
606
|
+
m4a: "m4a";
|
|
607
|
+
aac: "aac";
|
|
608
|
+
opus: "opus";
|
|
609
|
+
ogg: "ogg";
|
|
610
|
+
ac3: "ac3";
|
|
611
|
+
wma: "wma";
|
|
612
|
+
mp2: "mp2";
|
|
613
|
+
}>;
|
|
614
|
+
mimeType: import("zod").ZodString;
|
|
615
|
+
extension: import("zod").ZodString;
|
|
616
|
+
}, import("zod/v4/core").$strict>>;
|
|
617
|
+
encodeError: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
618
|
+
format: import("zod").ZodEnum<{
|
|
619
|
+
wav: "wav";
|
|
620
|
+
flac: "flac";
|
|
621
|
+
alac: "alac";
|
|
622
|
+
aiff: "aiff";
|
|
623
|
+
caf: "caf";
|
|
624
|
+
m4a: "m4a";
|
|
625
|
+
aac: "aac";
|
|
626
|
+
opus: "opus";
|
|
627
|
+
ogg: "ogg";
|
|
628
|
+
ac3: "ac3";
|
|
629
|
+
wma: "wma";
|
|
630
|
+
mp2: "mp2";
|
|
631
|
+
}>;
|
|
632
|
+
details: import("zod").ZodOptional<import("zod").ZodString>;
|
|
633
|
+
}, import("zod/v4/core").$strict>>;
|
|
512
634
|
done: import("zod").ZodDefault<import("zod").ZodBoolean>;
|
|
513
635
|
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
514
636
|
cancelled: "cancelled";
|
|
@@ -520,6 +642,7 @@ export declare const audioGenPlugin: {
|
|
|
520
642
|
realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
521
643
|
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
522
644
|
backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
645
|
+
gpuFallbackReason: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
523
646
|
emittedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
524
647
|
arMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
525
648
|
conditionMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -557,16 +680,16 @@ export declare const audioGenPlugin: {
|
|
|
557
680
|
gpu: "gpu";
|
|
558
681
|
}>;
|
|
559
682
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
560
|
-
cuda: "cuda";
|
|
561
683
|
vulkan: "vulkan";
|
|
562
|
-
metal: "metal";
|
|
563
684
|
opencl: "opencl";
|
|
564
|
-
rocm: "rocm";
|
|
565
685
|
opengl: "opengl";
|
|
566
686
|
webgpu: "webgpu";
|
|
687
|
+
metal: "metal";
|
|
567
688
|
direct3d11: "direct3d11";
|
|
568
689
|
direct3d12: "direct3d12";
|
|
690
|
+
cuda: "cuda";
|
|
569
691
|
levelZero: "levelZero";
|
|
692
|
+
rocm: "rocm";
|
|
570
693
|
}>>;
|
|
571
694
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
572
695
|
name: import("zod").ZodString;
|
|
@@ -602,13 +725,14 @@ export declare const audioGenPlugin: {
|
|
|
602
725
|
value: import("zod").ZodString;
|
|
603
726
|
}, import("zod/v4/core").$strip>], "type">;
|
|
604
727
|
type: import("zod").ZodLiteral<"audioUnderstand">;
|
|
605
|
-
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
606
728
|
modelId: import("zod").ZodString;
|
|
607
729
|
seed: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
608
730
|
vocalLanguage: import("zod").ZodOptional<import("zod").ZodString>;
|
|
609
731
|
lmTemperature: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
610
732
|
lmTopP: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
611
733
|
lmTopK: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
734
|
+
sessionId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
735
|
+
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
612
736
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
613
737
|
type: import("zod").ZodLiteral<"audioUnderstand">;
|
|
614
738
|
progress: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -637,6 +761,7 @@ export declare const audioGenPlugin: {
|
|
|
637
761
|
realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
638
762
|
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
639
763
|
backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
764
|
+
gpuFallbackReason: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
640
765
|
emittedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
641
766
|
arMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
642
767
|
conditionMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -674,16 +799,16 @@ export declare const audioGenPlugin: {
|
|
|
674
799
|
gpu: "gpu";
|
|
675
800
|
}>;
|
|
676
801
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
677
|
-
cuda: "cuda";
|
|
678
802
|
vulkan: "vulkan";
|
|
679
|
-
metal: "metal";
|
|
680
803
|
opencl: "opencl";
|
|
681
|
-
rocm: "rocm";
|
|
682
804
|
opengl: "opengl";
|
|
683
805
|
webgpu: "webgpu";
|
|
806
|
+
metal: "metal";
|
|
684
807
|
direct3d11: "direct3d11";
|
|
685
808
|
direct3d12: "direct3d12";
|
|
809
|
+
cuda: "cuda";
|
|
686
810
|
levelZero: "levelZero";
|
|
811
|
+
rocm: "rocm";
|
|
687
812
|
}>>;
|
|
688
813
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
689
814
|
name: import("zod").ZodString;
|
|
@@ -2,6 +2,7 @@ import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX, assessFit as audiogenAssessFi
|
|
|
2
2
|
import { ADDON_AUDIOGEN, ModelType, audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
4
|
import { resolveAudioGenConfig } from '../../../plugins/builtin/audiogen-ggml/config.js';
|
|
5
|
+
import { audiogenFitRequest } from '../../../plugins/builtin/audiogen-ggml/fit.js';
|
|
5
6
|
import { audioEditStream } from '../../../plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js';
|
|
6
7
|
import { audioGenStream } from '../../../plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js';
|
|
7
8
|
import { audioUnderstand } from '../../../plugins/builtin/audiogen-ggml/ops/audio-understand.js';
|
|
@@ -11,6 +12,7 @@ export const audioGenPlugin = definePlugin({
|
|
|
11
12
|
displayName: 'Audio Generation (GGML / ACE-Step and MiniMax)',
|
|
12
13
|
addonPackage: ADDON_AUDIOGEN,
|
|
13
14
|
loadConfigSchema: audioGenConfigSchema,
|
|
15
|
+
fitRequest: audiogenFitRequest,
|
|
14
16
|
assessFit: audiogenAssessFit,
|
|
15
17
|
// AudioGen's primary `modelSrc` is intentionally empty: all required
|
|
16
18
|
// weights are config-owned artifacts (ACE-Step: four sources; MiniMax: LM + synth).
|
|
@@ -82,6 +84,13 @@ function getAudioGenAddonConfig(config) {
|
|
|
82
84
|
return {
|
|
83
85
|
...commonConfig,
|
|
84
86
|
...(config.device !== undefined && { device: config.device }),
|
|
87
|
+
...(config.coremlConditionerPath !== undefined && {
|
|
88
|
+
coremlConditionerPath: config.coremlConditionerPath
|
|
89
|
+
}),
|
|
90
|
+
...(config.coremlProfileDir !== undefined && { coremlProfileDir: config.coremlProfileDir }),
|
|
91
|
+
...(config.coremlComputeUnits !== undefined && {
|
|
92
|
+
coremlComputeUnits: config.coremlComputeUnits
|
|
93
|
+
}),
|
|
85
94
|
...(config.inferenceSteps !== undefined && { inferenceSteps: config.inferenceSteps }),
|
|
86
95
|
...(config.cfgScale !== undefined && { cfgScale: config.cfgScale })
|
|
87
96
|
};
|
|
@@ -90,6 +99,9 @@ function getAudioGenAddonConfig(config) {
|
|
|
90
99
|
...commonConfig,
|
|
91
100
|
...(config.inferenceSteps !== undefined && { inferenceSteps: config.inferenceSteps }),
|
|
92
101
|
...(config.shift !== undefined && { shift: config.shift }),
|
|
93
|
-
...(config.nGpuLayers !== undefined && { nGpuLayers: config.nGpuLayers })
|
|
102
|
+
...(config.nGpuLayers !== undefined && { nGpuLayers: config.nGpuLayers }),
|
|
103
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
104
|
+
...(config.lmBackend !== undefined && { lmBackend: config.lmBackend }),
|
|
105
|
+
...(config.dumpStagesDir !== undefined && { dumpStagesDir: config.dumpStagesDir })
|
|
94
106
|
};
|
|
95
107
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/gpu.js';
|
|
2
|
+
import { audioSeconds, decoderCount } from '../../../plugins/builtin/asr-ggml/fit.js';
|
|
3
|
+
/** The BCI fit request for a load, with its embedder and decoder count. */
|
|
4
|
+
export function bciFitRequest(load) {
|
|
5
|
+
const config = (load.modelConfig ?? {});
|
|
6
|
+
const embedderPath = load.artifacts?.['embedderPath'];
|
|
7
|
+
const whisper = config.whisperConfig;
|
|
8
|
+
const decoders = decoderCount([whisper?.beam_search_beam_size, whisper?.greedy_best_of]);
|
|
9
|
+
const seconds = audioSeconds(whisper?.duration_ms);
|
|
10
|
+
const request = {
|
|
11
|
+
modelPath: load.modelPath,
|
|
12
|
+
...(embedderPath !== undefined && { embedderPath }),
|
|
13
|
+
// The load runs on the GPU unless told otherwise.
|
|
14
|
+
gpuLayers: gpuLayersFromGate(config.contextParams?.use_gpu ?? true),
|
|
15
|
+
...(config.contextParams?.gpu_device !== undefined && {
|
|
16
|
+
gpuDevice: config.contextParams.gpu_device
|
|
17
|
+
}),
|
|
18
|
+
...(decoders !== undefined && { decoders }),
|
|
19
|
+
...(seconds !== undefined && { audioSeconds: seconds })
|
|
20
|
+
};
|
|
21
|
+
return { request };
|
|
22
|
+
}
|