@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import type ASRGgml from '@qvac/asr-ggml';
|
|
2
2
|
import type { ParakeetConfig, WhisperConfig, MossTranscribeConfig } from '../../../schemas/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Artifact key under which the parakeet plugin's `resolveConfig` stores the
|
|
5
|
+
* resolved `speakerAttribution.modelSrc`; the addon wants that path, not the
|
|
6
|
+
* source it was resolved from.
|
|
7
|
+
*/
|
|
8
|
+
export declare const SPEAKER_ATTRIBUTION_MODEL_PATH = "speakerAttributionModelPath";
|
|
3
9
|
export declare function buildWhisperEngineConfig(config: WhisperConfig): {
|
|
4
10
|
miscConfig?: {
|
|
5
11
|
[k: string]: unknown;
|
|
@@ -10,11 +16,15 @@ export declare function buildWhisperEngineConfig(config: WhisperConfig): {
|
|
|
10
16
|
engine: "whisper";
|
|
11
17
|
whisperConfig: ASRGgml.WhisperConfig;
|
|
12
18
|
};
|
|
13
|
-
|
|
19
|
+
/**
|
|
20
|
+
* The addon-facing parakeet config. `speakerAttribution.modelSrc` is a model
|
|
21
|
+
* source the SDK resolves before the load; the addon takes the resolved path
|
|
22
|
+
* as `speakerAttribution.modelPath`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function buildParakeetConfig(config: ParakeetConfig, artifacts?: Record<string, string>): ASRGgml.ParakeetConfig;
|
|
25
|
+
export declare function buildParakeetEngineConfig(config: ParakeetConfig, artifacts?: Record<string, string>): {
|
|
14
26
|
engine: "parakeet";
|
|
15
|
-
parakeetConfig:
|
|
16
|
-
[k: string]: unknown;
|
|
17
|
-
};
|
|
27
|
+
parakeetConfig: import("node_modules/@qvac/asr-ggml/engines/parakeet/driver").ParakeetConfig;
|
|
18
28
|
};
|
|
19
29
|
export declare function buildWhisperReloadConfig(config: WhisperConfig): {
|
|
20
30
|
miscConfig?: {
|
|
@@ -24,6 +34,11 @@ export declare function buildWhisperReloadConfig(config: WhisperConfig): {
|
|
|
24
34
|
[k: string]: unknown;
|
|
25
35
|
};
|
|
26
36
|
};
|
|
37
|
+
/**
|
|
38
|
+
* A reload merges into the loaded config, so leaving `speakerAttribution` out
|
|
39
|
+
* keeps the diarization model the load resolved; the reload path has no
|
|
40
|
+
* artifacts to resolve a new source with.
|
|
41
|
+
*/
|
|
27
42
|
export declare function buildParakeetReloadConfig(config: ParakeetConfig): {
|
|
28
43
|
parakeetConfig: {
|
|
29
44
|
[k: string]: unknown;
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import { ModelLoadFailedError } from '../../../errors/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Artifact key under which the parakeet plugin's `resolveConfig` stores the
|
|
4
|
+
* resolved `speakerAttribution.modelSrc`; the addon wants that path, not the
|
|
5
|
+
* source it was resolved from.
|
|
6
|
+
*/
|
|
7
|
+
export const SPEAKER_ATTRIBUTION_MODEL_PATH = 'speakerAttributionModelPath';
|
|
1
8
|
function omitUndefined(config) {
|
|
2
9
|
return Object.fromEntries(Object.entries(config).filter(([, value]) => value !== undefined));
|
|
3
10
|
}
|
|
@@ -24,10 +31,33 @@ export function buildWhisperEngineConfig(config) {
|
|
|
24
31
|
...(config.miscConfig && { miscConfig: omitUndefined(config.miscConfig) })
|
|
25
32
|
};
|
|
26
33
|
}
|
|
27
|
-
|
|
34
|
+
/**
|
|
35
|
+
* The addon-facing parakeet config. `speakerAttribution.modelSrc` is a model
|
|
36
|
+
* source the SDK resolves before the load; the addon takes the resolved path
|
|
37
|
+
* as `speakerAttribution.modelPath`.
|
|
38
|
+
*/
|
|
39
|
+
export function buildParakeetConfig(config, artifacts) {
|
|
40
|
+
const { speakerAttribution, ...rest } = config;
|
|
41
|
+
const parakeetConfig = omitUndefined(rest);
|
|
42
|
+
if (speakerAttribution === undefined)
|
|
43
|
+
return parakeetConfig;
|
|
44
|
+
const modelPath = artifacts?.[SPEAKER_ATTRIBUTION_MODEL_PATH];
|
|
45
|
+
if (!modelPath) {
|
|
46
|
+
throw new ModelLoadFailedError('speakerAttribution.modelSrc was not resolved to a diarization model path');
|
|
47
|
+
}
|
|
48
|
+
const { modelSrc: _modelSrc, ...options } = speakerAttribution;
|
|
49
|
+
return {
|
|
50
|
+
...parakeetConfig,
|
|
51
|
+
speakerAttribution: {
|
|
52
|
+
modelPath,
|
|
53
|
+
...omitUndefined(options)
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export function buildParakeetEngineConfig(config, artifacts) {
|
|
28
58
|
return {
|
|
29
59
|
engine: 'parakeet',
|
|
30
|
-
parakeetConfig:
|
|
60
|
+
parakeetConfig: buildParakeetConfig(config, artifacts)
|
|
31
61
|
};
|
|
32
62
|
}
|
|
33
63
|
export function buildWhisperReloadConfig(config) {
|
|
@@ -39,9 +69,15 @@ export function buildWhisperReloadConfig(config) {
|
|
|
39
69
|
})
|
|
40
70
|
};
|
|
41
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* A reload merges into the loaded config, so leaving `speakerAttribution` out
|
|
74
|
+
* keeps the diarization model the load resolved; the reload path has no
|
|
75
|
+
* artifacts to resolve a new source with.
|
|
76
|
+
*/
|
|
42
77
|
export function buildParakeetReloadConfig(config) {
|
|
78
|
+
const { speakerAttribution: _speakerAttribution, ...rest } = config;
|
|
43
79
|
return {
|
|
44
|
-
parakeetConfig: omitUndefined(
|
|
80
|
+
parakeetConfig: omitUndefined(rest)
|
|
45
81
|
};
|
|
46
82
|
}
|
|
47
83
|
export function buildMossTranscribeEngineConfig(config) {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { FitLoad, FitRequestPlan } from '../../../schemas/plugin.js';
|
|
2
|
+
/**
|
|
3
|
+
* The longest single transcribe the projection has to cover. Whisper's KV cache
|
|
4
|
+
* and decode graph are sized from it; unset leaves the engine's own default,
|
|
5
|
+
* which is the worst case it supports.
|
|
6
|
+
*/
|
|
7
|
+
export declare function audioSeconds(durationMs: number | undefined): number | undefined;
|
|
8
|
+
/**
|
|
9
|
+
* Worst-case resident decoders. Each strategy carries its own count, and `-1`
|
|
10
|
+
* is the engine's "use the default" sentinel, not a number of decoders.
|
|
11
|
+
*/
|
|
12
|
+
export declare function decoderCount(candidates: readonly (number | undefined)[]): number | undefined;
|
|
13
|
+
/** The MOSS-Transcribe fit request for a load, sized by the declared recording. */
|
|
14
|
+
export declare function mossTranscribeFitRequest(load: FitLoad): FitRequestPlan;
|
|
15
|
+
/** The Whisper fit request for a load, with its VAD model and decoder count. */
|
|
16
|
+
export declare function whisperFitRequest(load: FitLoad): FitRequestPlan;
|
|
17
|
+
/** The Parakeet fit request for a load, covering its live session when it streams. */
|
|
18
|
+
export declare function parakeetFitRequest(load: FitLoad): FitRequestPlan;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/gpu.js';
|
|
2
|
+
/**
|
|
3
|
+
* The longest single transcribe the projection has to cover. Whisper's KV cache
|
|
4
|
+
* and decode graph are sized from it; unset leaves the engine's own default,
|
|
5
|
+
* which is the worst case it supports.
|
|
6
|
+
*/
|
|
7
|
+
export function audioSeconds(durationMs) {
|
|
8
|
+
if (durationMs === undefined || durationMs <= 0)
|
|
9
|
+
return undefined;
|
|
10
|
+
return durationMs / 1000;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Worst-case resident decoders. Each strategy carries its own count, and `-1`
|
|
14
|
+
* is the engine's "use the default" sentinel, not a number of decoders.
|
|
15
|
+
*/
|
|
16
|
+
export function decoderCount(candidates) {
|
|
17
|
+
for (const candidate of candidates) {
|
|
18
|
+
if (candidate !== undefined && candidate > 0)
|
|
19
|
+
return candidate;
|
|
20
|
+
}
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
/** The MOSS-Transcribe fit request for a load, sized by the declared recording. */
|
|
24
|
+
export function mossTranscribeFitRequest(load) {
|
|
25
|
+
const workload = load.transcriptionWorkload;
|
|
26
|
+
if (workload === undefined) {
|
|
27
|
+
return { unknown: 'a MOSS projection needs a transcriptionWorkload with its audioSeconds' };
|
|
28
|
+
}
|
|
29
|
+
const config = (load.modelConfig ?? {});
|
|
30
|
+
const request = {
|
|
31
|
+
engine: 'moss-transcribe',
|
|
32
|
+
modelPath: load.modelPath,
|
|
33
|
+
audioSeconds: workload.audioSeconds,
|
|
34
|
+
...(workload.prompt !== undefined && { prompt: workload.prompt }),
|
|
35
|
+
...(workload.hotwords !== undefined && { hotwords: workload.hotwords }),
|
|
36
|
+
...(workload.maxNewTokens !== undefined && { maxNewTokens: workload.maxNewTokens }),
|
|
37
|
+
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
38
|
+
// An explicit selector projects on that device and overrides the GPU gate,
|
|
39
|
+
// as it does at load; a missing device rejects the fit.
|
|
40
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
41
|
+
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
42
|
+
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir })
|
|
43
|
+
};
|
|
44
|
+
return { request };
|
|
45
|
+
}
|
|
46
|
+
/** The Whisper fit request for a load, with its VAD model and decoder count. */
|
|
47
|
+
export function whisperFitRequest(load) {
|
|
48
|
+
const config = (load.modelConfig ?? {});
|
|
49
|
+
const vadModelPath = load.artifacts?.['vadModelPath'];
|
|
50
|
+
const decoders = decoderCount([
|
|
51
|
+
config.strategy === 'beam_search' ? config.beam_search_beam_size : config.greedy_best_of
|
|
52
|
+
]);
|
|
53
|
+
const seconds = audioSeconds(config.duration_ms);
|
|
54
|
+
const request = {
|
|
55
|
+
engine: 'whisper',
|
|
56
|
+
modelPath: load.modelPath,
|
|
57
|
+
gpuLayers: gpuLayersFromGate(config.contextParams?.use_gpu),
|
|
58
|
+
...(vadModelPath !== undefined && { vadModelPath }),
|
|
59
|
+
...(config.contextParams?.flash_attn !== undefined && {
|
|
60
|
+
flashAttn: config.contextParams.flash_attn
|
|
61
|
+
}),
|
|
62
|
+
...(config.contextParams?.gpu_device !== undefined && {
|
|
63
|
+
gpuDevice: config.contextParams.gpu_device
|
|
64
|
+
}),
|
|
65
|
+
...(decoders !== undefined && { decoders }),
|
|
66
|
+
...(seconds !== undefined && { audioSeconds: seconds })
|
|
67
|
+
};
|
|
68
|
+
return { request };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The encoder's left context and Sortformer's rolling history size its live
|
|
72
|
+
* state, and the fit request measures that state in frames while the load sets
|
|
73
|
+
* it in milliseconds. Converting needs the model's frame rate, a property of
|
|
74
|
+
* the checkpoint, so a streaming load that pins either is not projected.
|
|
75
|
+
*/
|
|
76
|
+
const UNREPRESENTABLE_STREAMING = [
|
|
77
|
+
'streamingLeftContextMs',
|
|
78
|
+
'streamingHistoryMs'
|
|
79
|
+
];
|
|
80
|
+
/** The Parakeet fit request for a load, covering its live session when it streams. */
|
|
81
|
+
export function parakeetFitRequest(load) {
|
|
82
|
+
const config = (load.modelConfig ?? {});
|
|
83
|
+
if (config.streaming === true) {
|
|
84
|
+
for (const key of UNREPRESENTABLE_STREAMING) {
|
|
85
|
+
if (config[key] !== undefined)
|
|
86
|
+
return { unknown: `the fitter cannot represent ${key}` };
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// The addon projects one model, and a speaker-attributed load holds two,
|
|
90
|
+
// so a one-model verdict would understate it.
|
|
91
|
+
if (config.speakerAttribution !== undefined) {
|
|
92
|
+
return {
|
|
93
|
+
unknown: 'the fitter projects one model; assess the speaker-attribution diarization model as its own load'
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
const request = {
|
|
97
|
+
engine: 'parakeet',
|
|
98
|
+
modelPath: load.modelPath,
|
|
99
|
+
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
100
|
+
// An explicit selector projects on that device and overrides the GPU gate,
|
|
101
|
+
// as it does at load; a missing device rejects the fit.
|
|
102
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
103
|
+
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
104
|
+
// The long-form window bounds the offline encoder graph the projection
|
|
105
|
+
// sizes, so an explicit window (or the one-pass sentinel) reaches the fit.
|
|
106
|
+
...(config.longFormWindowFrames !== undefined && {
|
|
107
|
+
longFormWindowFrames: config.longFormWindowFrames
|
|
108
|
+
}),
|
|
109
|
+
...(config.longFormContextFrames !== undefined && {
|
|
110
|
+
longFormContextFrames: config.longFormContextFrames
|
|
111
|
+
}),
|
|
112
|
+
// Only a streaming load holds a live session, so outside one the chunk
|
|
113
|
+
// cadence describes no allocation the projection should cover.
|
|
114
|
+
...(config.streaming === true &&
|
|
115
|
+
config.streamingChunkMs !== undefined && { nemotronChunkMs: config.streamingChunkMs })
|
|
116
|
+
};
|
|
117
|
+
return { request };
|
|
118
|
+
}
|
|
@@ -3,6 +3,9 @@ export declare function resolveAudioGenConfig(config: Record<string, unknown>, c
|
|
|
3
3
|
config: {
|
|
4
4
|
engine: "minimax";
|
|
5
5
|
device?: "cpu" | "gpu" | "auto" | undefined;
|
|
6
|
+
coremlProfileDir?: string | undefined;
|
|
7
|
+
coremlComputeUnits?: "CPU_ONLY" | "CPU_AND_GPU" | "CPU_AND_NE" | "ALL" | undefined;
|
|
8
|
+
coremlConditionerPath?: string | undefined;
|
|
6
9
|
inferenceSteps?: number | undefined;
|
|
7
10
|
cfgScale?: number | undefined;
|
|
8
11
|
useGPU?: boolean | undefined;
|
|
@@ -18,6 +21,9 @@ export declare function resolveAudioGenConfig(config: Record<string, unknown>, c
|
|
|
18
21
|
inferenceSteps?: number | undefined;
|
|
19
22
|
shift?: number | undefined;
|
|
20
23
|
nGpuLayers?: number | undefined;
|
|
24
|
+
backend?: string | undefined;
|
|
25
|
+
lmBackend?: string | undefined;
|
|
26
|
+
dumpStagesDir?: string | undefined;
|
|
21
27
|
useGPU?: boolean | undefined;
|
|
22
28
|
threads?: number | undefined;
|
|
23
29
|
backendsDir?: string | undefined;
|
|
@@ -10,6 +10,15 @@ async function resolveMinimaxConfig(parsed, ctx) {
|
|
|
10
10
|
if (ctx.platform === 'android' || ctx.platform === 'ios') {
|
|
11
11
|
throw new ModelLoadFailedError('MiniMax-Music3 is available on desktop only');
|
|
12
12
|
}
|
|
13
|
+
// Refused before the model pair is resolved: the addon would only say so
|
|
14
|
+
// once several gigabytes had been downloaded. Only the platform is known
|
|
15
|
+
// here; the addon still checks the macOS version at load.
|
|
16
|
+
if (parsed.coremlProfileDir !== undefined && ctx.platform !== 'darwin') {
|
|
17
|
+
throw new ModelLoadFailedError('MiniMax Core ML is available on macOS only');
|
|
18
|
+
}
|
|
19
|
+
if (parsed.coremlConditionerPath !== undefined && ctx.platform !== 'darwin') {
|
|
20
|
+
throw new ModelLoadFailedError('The MiniMax Core ML conditioner is available on macOS only');
|
|
21
|
+
}
|
|
13
22
|
const { lmModelSrc, synthModelSrc, ...runtimeConfig } = parsed;
|
|
14
23
|
// Resolve sequentially for the same stall-timeout reason documented for the
|
|
15
24
|
// larger ACE-Step artifact set below.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from '
|
|
1
|
+
import { gpuLayersFromGate } from '../../../model-fit/gpu.js';
|
|
2
2
|
/**
|
|
3
3
|
* `@qvac/audiogen-ggml` is an optional peer dependency, so its value exports
|
|
4
4
|
* are unavailable to a host that ships without it. The engine discriminator is
|
|
@@ -12,14 +12,15 @@ const MINIMAX = 'minimax';
|
|
|
12
12
|
*
|
|
13
13
|
* `device` is passed through as the load takes it. Without one, the GPU switch
|
|
14
14
|
* maps to `auto` as it does on load: the fitter reads a positive layer count
|
|
15
|
-
* that way.
|
|
16
|
-
*
|
|
15
|
+
* that way. A Core ML profile or conditioner is passed through too: the fitter
|
|
16
|
+
* cannot size either route and answers `unknown` for it. The plugin refuses
|
|
17
|
+
* MiniMax on mobile before this is reached.
|
|
17
18
|
*/
|
|
18
|
-
function minimaxFitRequest(config, artifacts
|
|
19
|
+
function minimaxFitRequest(config, artifacts) {
|
|
19
20
|
const lmPath = artifacts['lmModelPath'];
|
|
20
21
|
const synthPath = artifacts['synthModelPath'];
|
|
21
22
|
if (lmPath === undefined || synthPath === undefined) {
|
|
22
|
-
return {
|
|
23
|
+
return { unknown: 'the minimax model pair is not resolved' };
|
|
23
24
|
}
|
|
24
25
|
const request = {
|
|
25
26
|
engine: MINIMAX,
|
|
@@ -28,17 +29,23 @@ function minimaxFitRequest(config, artifacts, marginBytes) {
|
|
|
28
29
|
...(config.device !== undefined
|
|
29
30
|
? { device: config.device }
|
|
30
31
|
: { gpuLayers: gpuLayersFromGate(config.useGPU) }),
|
|
31
|
-
...(config.
|
|
32
|
-
|
|
32
|
+
...(config.coremlConditionerPath !== undefined && {
|
|
33
|
+
coremlConditionerPath: config.coremlConditionerPath
|
|
34
|
+
}),
|
|
35
|
+
...(config.coremlProfileDir !== undefined && { coremlProfileDir: config.coremlProfileDir }),
|
|
36
|
+
...(config.coremlComputeUnits !== undefined && {
|
|
37
|
+
coremlComputeUnits: config.coremlComputeUnits
|
|
38
|
+
}),
|
|
39
|
+
...(config.threads !== undefined && { threads: config.threads })
|
|
33
40
|
};
|
|
34
|
-
return {
|
|
41
|
+
return { request };
|
|
35
42
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
43
|
+
/** The audiogen fit request for a load, from its resolved stage checkpoints. */
|
|
44
|
+
export function audiogenFitRequest(load) {
|
|
45
|
+
const config = (load.modelConfig ?? {});
|
|
46
|
+
const artifacts = load.artifacts ?? {};
|
|
47
|
+
if (config.engine === MINIMAX)
|
|
48
|
+
return minimaxFitRequest(config, artifacts);
|
|
42
49
|
const textEncoderPath = artifacts['textEncModelPath'];
|
|
43
50
|
const lmPath = artifacts['lmModelPath'];
|
|
44
51
|
const ditPath = artifacts['ditModelPath'];
|
|
@@ -47,17 +54,20 @@ export function createAudiogenFitRequest(params) {
|
|
|
47
54
|
lmPath === undefined ||
|
|
48
55
|
ditPath === undefined ||
|
|
49
56
|
vaePath === undefined) {
|
|
50
|
-
return {
|
|
57
|
+
return { unknown: 'the ace-step stage checkpoints are not all resolved' };
|
|
51
58
|
}
|
|
52
59
|
const layers = 'nGpuLayers' in config ? config.nGpuLayers : undefined;
|
|
60
|
+
// A named backend moves a stage to its own device, so the fitter is given
|
|
61
|
+
// the placement the load will ask for and sizes each device it names.
|
|
53
62
|
const request = {
|
|
54
63
|
textEncoderPath,
|
|
55
64
|
lmPath,
|
|
56
65
|
ditPath,
|
|
57
66
|
vaePath,
|
|
58
67
|
gpuLayers: gpuLayersFromGate(config.useGPU, layers),
|
|
59
|
-
...(config.
|
|
60
|
-
...(
|
|
68
|
+
...(config.backend !== undefined && { backend: config.backend }),
|
|
69
|
+
...(config.lmBackend !== undefined && { lmBackend: config.lmBackend }),
|
|
70
|
+
...(config.threads !== undefined && { threads: config.threads })
|
|
61
71
|
};
|
|
62
|
-
return {
|
|
72
|
+
return { request };
|
|
63
73
|
}
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
+
import type Buffer from 'bare-buffer';
|
|
2
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
3
|
import { type AudioEditStreamRequest, type AudioEditStreamResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
-
export declare function audioEditStream(request: AudioEditStreamRequest): AsyncGenerator<AudioEditStreamResponse>;
|
|
4
|
+
export declare function audioEditStream(request: AudioEditStreamRequest, _inputStream?: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<AudioEditStreamResponse>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { RepaintMode } from '@qvac/audiogen-ggml';
|
|
2
2
|
import { AUDIOGEN_INPUT_CHANNELS, AUDIOGEN_INPUT_SAMPLE_RATE } from '../../../../schemas/audio-gen.js';
|
|
3
|
-
import { resolveAudioGenInputs } from '
|
|
4
|
-
import { streamAudioGenRun } from '
|
|
3
|
+
import { resolveAudioGenInputs } from '../../../../plugins/builtin/audiogen-ggml/ops/audio-gen-input.js';
|
|
4
|
+
import { streamAudioGenRun } from '../../../../plugins/builtin/audiogen-ggml/ops/audio-gen-run.js';
|
|
5
5
|
// The wire vocabulary is the addon's own enum values; the map keeps the
|
|
6
6
|
// string-enum typing honest instead of casting across the boundary.
|
|
7
7
|
const REPAINT_MODES = {
|
|
@@ -9,10 +9,11 @@ const REPAINT_MODES = {
|
|
|
9
9
|
balanced: RepaintMode.Balanced,
|
|
10
10
|
aggressive: RepaintMode.Aggressive
|
|
11
11
|
};
|
|
12
|
-
export function audioEditStream(request) {
|
|
12
|
+
export function audioEditStream(request, _inputStream, context) {
|
|
13
13
|
return streamAudioGenRun({
|
|
14
14
|
type: 'audioEditStream',
|
|
15
15
|
request,
|
|
16
|
+
context,
|
|
16
17
|
async start(model, ctx, logger) {
|
|
17
18
|
// The source and reference are decoded before the run is admitted so
|
|
18
19
|
// the model slot is never held by a request that fails on input
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { AudioGenOutputFormat, AudioGenStreamResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
+
export interface PcmLayout {
|
|
3
|
+
sampleRate: number;
|
|
4
|
+
channels: number;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Proves the host can encode `format` before a generation is admitted: the
|
|
8
|
+
* FFmpeg build is the addon's, and a missing encoder would otherwise only
|
|
9
|
+
* surface after the run had finished. `wav` is written in JS and needs no
|
|
10
|
+
* probe. The outcome is remembered per format, so only the first request for
|
|
11
|
+
* a format pays for the probe.
|
|
12
|
+
*/
|
|
13
|
+
export declare function assertOutputFormatAvailable(format: AudioGenOutputFormat): void;
|
|
14
|
+
/** Forgets every probe outcome. For tests that swap the encoder. */
|
|
15
|
+
export declare function resetOutputFormatProbes(): void;
|
|
16
|
+
/** Encodes a run's interleaved Int16 PCM into the frame field that carries it. */
|
|
17
|
+
export declare function encodeRunAudio(chunks: readonly Uint8Array[], format: AudioGenOutputFormat, layout: PcmLayout): NonNullable<AudioGenStreamResponse['encoded']>;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { AudioGen } from '@qvac/audiogen-ggml';
|
|
2
|
+
import { AudioEncodeFailedError } from '../../../../errors/index.js';
|
|
3
|
+
// A hundredth of a second of stereo silence at 48 kHz: enough for every
|
|
4
|
+
// encoder to open, emit a frame and close its container.
|
|
5
|
+
const PROBE_SAMPLE_RATE = 48000;
|
|
6
|
+
const PROBE_CHANNELS = 2;
|
|
7
|
+
const PROBE_FRAMES = 480;
|
|
8
|
+
function encode(pcm, format, layout) {
|
|
9
|
+
try {
|
|
10
|
+
return AudioGen.encode(pcm, format, layout);
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
throw new AudioEncodeFailedError(format, error instanceof Error ? error.message : String(error), error);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
// The addon's FFmpeg build is fixed for the life of the process, so each
|
|
17
|
+
// format is probed once: either the encoder is there, or the entry keeps what
|
|
18
|
+
// the failed probe reported so the same error can be raised again.
|
|
19
|
+
const probeResults = new Map();
|
|
20
|
+
/**
|
|
21
|
+
* Proves the host can encode `format` before a generation is admitted: the
|
|
22
|
+
* FFmpeg build is the addon's, and a missing encoder would otherwise only
|
|
23
|
+
* surface after the run had finished. `wav` is written in JS and needs no
|
|
24
|
+
* probe. The outcome is remembered per format, so only the first request for
|
|
25
|
+
* a format pays for the probe.
|
|
26
|
+
*/
|
|
27
|
+
export function assertOutputFormatAvailable(format) {
|
|
28
|
+
if (format === 'wav')
|
|
29
|
+
return;
|
|
30
|
+
const known = probeResults.get(format);
|
|
31
|
+
if (known === 'available')
|
|
32
|
+
return;
|
|
33
|
+
if (known !== undefined)
|
|
34
|
+
throw new AudioEncodeFailedError(format, known.details);
|
|
35
|
+
try {
|
|
36
|
+
encode(new Uint8Array(PROBE_FRAMES * PROBE_CHANNELS * Int16Array.BYTES_PER_ELEMENT), format, {
|
|
37
|
+
sampleRate: PROBE_SAMPLE_RATE,
|
|
38
|
+
channels: PROBE_CHANNELS
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
probeResults.set(format, {
|
|
43
|
+
details: error instanceof AudioEncodeFailedError ? error.details : String(error)
|
|
44
|
+
});
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
probeResults.set(format, 'available');
|
|
48
|
+
}
|
|
49
|
+
/** Forgets every probe outcome. For tests that swap the encoder. */
|
|
50
|
+
export function resetOutputFormatProbes() {
|
|
51
|
+
probeResults.clear();
|
|
52
|
+
}
|
|
53
|
+
/** Encodes a run's interleaved Int16 PCM into the frame field that carries it. */
|
|
54
|
+
export function encodeRunAudio(chunks, format, layout) {
|
|
55
|
+
const pcm = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.byteLength, 0));
|
|
56
|
+
let offset = 0;
|
|
57
|
+
for (const chunk of chunks) {
|
|
58
|
+
pcm.set(chunk, offset);
|
|
59
|
+
offset += chunk.byteLength;
|
|
60
|
+
}
|
|
61
|
+
const encoded = encode(pcm, format, layout);
|
|
62
|
+
return {
|
|
63
|
+
data: Buffer.from(encoded.data.buffer, encoded.data.byteOffset, encoded.data.byteLength).toString('base64'),
|
|
64
|
+
format,
|
|
65
|
+
mimeType: encoded.mimeType,
|
|
66
|
+
extension: encoded.extension
|
|
67
|
+
};
|
|
68
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
2
|
import { AudioGen } from '@qvac/audiogen-ggml';
|
|
2
|
-
import { type AudioGenStreamResponse, type AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
|
|
3
|
+
import { type AudioGenOutputFormat, type AudioGenStreamResponse, type AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
|
|
3
4
|
import { type Logger } from '../../../../logging/index.js';
|
|
4
5
|
import { type RequestContext } from '../../../../runtime/index.js';
|
|
5
6
|
/**
|
|
@@ -26,7 +27,12 @@ export interface AudioGenRunOptions<TType extends AudioGenRunType> {
|
|
|
26
27
|
request: {
|
|
27
28
|
modelId: string;
|
|
28
29
|
requestId?: string | undefined;
|
|
30
|
+
sessionId?: string | undefined;
|
|
31
|
+
/** Generation and editing only: also deliver the run in this container. */
|
|
32
|
+
outputFormat?: AudioGenOutputFormat | undefined;
|
|
29
33
|
};
|
|
34
|
+
/** The invoke this run belongs to, when reached through `pluginInvoke`. */
|
|
35
|
+
context?: PluginHandlerContext | undefined;
|
|
30
36
|
/**
|
|
31
37
|
* Prepares the inputs and admits the native job once the request holds the
|
|
32
38
|
* model slot. Preparation (decoding audio) can take real time, so the
|
|
@@ -39,8 +45,10 @@ export interface AudioGenRunOptions<TType extends AudioGenRunType> {
|
|
|
39
45
|
}
|
|
40
46
|
/**
|
|
41
47
|
* Runs one AudioGen job under the request registry and streams it as
|
|
42
|
-
* `TType` frames: progress ticks, PCM chunks,
|
|
43
|
-
*
|
|
48
|
+
* `TType` frames: progress ticks, PCM chunks, the encoded file (or the reason
|
|
49
|
+
* it could not be encoded) when the request named an `outputFormat`, then the
|
|
50
|
+
* terminal frame with stats and
|
|
51
|
+
* backend diagnostics. Owns admission (`kind: 'audiogen'`, one
|
|
44
52
|
* active job per model), abort-to-cancel wiring, and the cancelled terminal.
|
|
45
53
|
*/
|
|
46
54
|
export declare function streamAudioGenRun<TType extends AudioGenRunType>(options: AudioGenRunOptions<TType>): AsyncGenerator<AudioGenRunFrame<TType>>;
|
|
@@ -50,8 +50,8 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../../../schemas/plugin.js';
|
|
53
54
|
import { AudioGen, audiogenBackendName, audiogenGpuFallbackReason } from '@qvac/audiogen-ggml';
|
|
54
|
-
import { z } from 'zod';
|
|
55
55
|
import { audioGenStatsSchema } from '../../../../schemas/audio-gen.js';
|
|
56
56
|
import { graphicsDriverSchema } from '../../../../schemas/index.js';
|
|
57
57
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
@@ -59,19 +59,28 @@ import { attachBackendDiagnostics } from '../../../../profiling/backend-diagnost
|
|
|
59
59
|
import { getModel } from '../../../../runtime/model-registry.js';
|
|
60
60
|
import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
|
|
61
61
|
import { generateRequestId } from '../../../../runtime/request-id.js';
|
|
62
|
-
import { ModelOperationNotSupportedError } from '../../../../errors/index.js';
|
|
62
|
+
import { AudioEncodeFailedError, ModelOperationNotSupportedError } from '../../../../errors/index.js';
|
|
63
|
+
import { assertOutputFormatAvailable, encodeRunAudio } from '../../../../plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js';
|
|
63
64
|
/**
|
|
64
65
|
* Runs one AudioGen job under the request registry and streams it as
|
|
65
|
-
* `TType` frames: progress ticks, PCM chunks,
|
|
66
|
-
*
|
|
66
|
+
* `TType` frames: progress ticks, PCM chunks, the encoded file (or the reason
|
|
67
|
+
* it could not be encoded) when the request named an `outputFormat`, then the
|
|
68
|
+
* terminal frame with stats and
|
|
69
|
+
* backend diagnostics. Owns admission (`kind: 'audiogen'`, one
|
|
67
70
|
* active job per model), abort-to-cancel wiring, and the cancelled terminal.
|
|
68
71
|
*/
|
|
69
72
|
export async function* streamAudioGenRun(options) {
|
|
70
73
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
71
74
|
try {
|
|
72
|
-
const { type, request } = options;
|
|
75
|
+
const { type, request, context } = options;
|
|
76
|
+
// Checked before the request queues for the model: an encoder the host lacks
|
|
77
|
+
// must not cost a whole generation to find out about.
|
|
78
|
+
if (request.outputFormat !== undefined)
|
|
79
|
+
assertOutputFormatAvailable(request.outputFormat);
|
|
73
80
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
74
81
|
requestId: request.requestId ?? generateRequestId(),
|
|
82
|
+
sessionId: request.sessionId,
|
|
83
|
+
...requestParent(context),
|
|
75
84
|
kind: 'audiogen',
|
|
76
85
|
modelId: request.modelId
|
|
77
86
|
}), true);
|
|
@@ -103,6 +112,9 @@ export async function* streamAudioGenRun(options) {
|
|
|
103
112
|
});
|
|
104
113
|
let response;
|
|
105
114
|
let streamedUnderstand;
|
|
115
|
+
// Kept only when the run is also to be encoded.
|
|
116
|
+
const encodeChunks = [];
|
|
117
|
+
let encodeLayout;
|
|
106
118
|
try {
|
|
107
119
|
response = await options.start(model, ctx, logger);
|
|
108
120
|
if (response !== undefined) {
|
|
@@ -129,6 +141,12 @@ export async function* streamAudioGenRun(options) {
|
|
|
129
141
|
if (!('outputArray' in chunk))
|
|
130
142
|
continue;
|
|
131
143
|
const pcm = new Uint8Array(chunk.outputArray.buffer, chunk.outputArray.byteOffset, chunk.outputArray.byteLength);
|
|
144
|
+
if (request.outputFormat !== undefined) {
|
|
145
|
+
// Copied: the view is over the addon's buffer, which is held here
|
|
146
|
+
// until the run ends and must not change under a later chunk.
|
|
147
|
+
encodeChunks.push(pcm.slice());
|
|
148
|
+
encodeLayout = { sampleRate: chunk.sampleRate, channels: chunk.channels };
|
|
149
|
+
}
|
|
132
150
|
yield {
|
|
133
151
|
type,
|
|
134
152
|
data: Buffer.from(pcm).toString('base64'),
|
|
@@ -148,6 +166,15 @@ export async function* streamAudioGenRun(options) {
|
|
|
148
166
|
yield { type, done: true, stopReason: 'cancelled' };
|
|
149
167
|
return;
|
|
150
168
|
}
|
|
169
|
+
if (request.outputFormat !== undefined && encodeLayout !== undefined) {
|
|
170
|
+
// The PCM is already with the caller. A failure here costs only the file:
|
|
171
|
+
// it is reported on its own frame and the run still ends as completed.
|
|
172
|
+
yield {
|
|
173
|
+
type,
|
|
174
|
+
...encodeOutcome(encodeChunks, request.outputFormat, encodeLayout, logger),
|
|
175
|
+
done: false
|
|
176
|
+
};
|
|
177
|
+
}
|
|
151
178
|
const raw = await response.await();
|
|
152
179
|
// `understand.audioCodes` arrives as an Int32Array, which the wire schema —
|
|
153
180
|
// and anything downstream of JSON — needs as a plain array of integers.
|
|
@@ -157,7 +184,7 @@ export async function* streamAudioGenRun(options) {
|
|
|
157
184
|
? { ...raw, understand: streamedUnderstand ?? toUnderstandResult(raw.understand) }
|
|
158
185
|
: raw;
|
|
159
186
|
const stats = audioGenStatsSchema.parse(normalized);
|
|
160
|
-
const diagnostics = buildBackendDiagnostics(
|
|
187
|
+
const diagnostics = buildBackendDiagnostics(stats);
|
|
161
188
|
const terminal = {
|
|
162
189
|
type,
|
|
163
190
|
done: true,
|
|
@@ -177,11 +204,20 @@ export async function* streamAudioGenRun(options) {
|
|
|
177
204
|
await result_1;
|
|
178
205
|
}
|
|
179
206
|
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
207
|
+
function encodeOutcome(chunks, format, layout, logger) {
|
|
208
|
+
try {
|
|
209
|
+
return { encoded: encodeRunAudio(chunks, format, layout) };
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
const details = error instanceof AudioEncodeFailedError
|
|
213
|
+
? error.details
|
|
214
|
+
: error instanceof Error
|
|
215
|
+
? error.message
|
|
216
|
+
: String(error);
|
|
217
|
+
logger.warn(`[audiogen] encoding the finished run as ${format} failed; delivering PCM only: ${details ?? 'no detail'}`);
|
|
218
|
+
return { encodeError: { format, ...(details !== undefined && { details }) } };
|
|
219
|
+
}
|
|
220
|
+
}
|
|
185
221
|
/**
|
|
186
222
|
* Normalizes the addon's understand result for the wire: `audioCodes` arrives
|
|
187
223
|
* as an `Int32Array`, which neither the schema nor a JSON transport accepts.
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
+
import type Buffer from 'bare-buffer';
|
|
2
|
+
import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
3
|
import type { AudioGenStreamRequest, AudioGenStreamResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
-
export declare function audioGenStream(request: AudioGenStreamRequest): AsyncGenerator<AudioGenStreamResponse>;
|
|
4
|
+
export declare function audioGenStream(request: AudioGenStreamRequest, _inputStream?: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<AudioGenStreamResponse>;
|