@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
package/dist/utils/tts-stats.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { encodeTtsPcm } from '../schemas/tts-pcm.js';
|
|
1
2
|
/**
|
|
2
3
|
* Everything an addon chunk carries beyond its PCM. Shared by both TTS ops so
|
|
3
4
|
* the plain `stream: true` path and the sentence-stream path emit the same
|
|
@@ -11,9 +12,19 @@ export function chunkMetadata(chunk) {
|
|
|
11
12
|
...(typeof chunk.sentenceChunk === 'string' && chunk.sentenceChunk.length > 0
|
|
12
13
|
? { sentenceChunk: chunk.sentenceChunk }
|
|
13
14
|
: {}),
|
|
14
|
-
...(chunk.isLast !== undefined ? { isLast: chunk.isLast } : {})
|
|
15
|
+
...(chunk.isLast !== undefined ? { isLast: chunk.isLast } : {}),
|
|
16
|
+
...(chunk.speechTokenIds !== undefined ? { speechTokenIds: chunk.speechTokenIds } : {})
|
|
15
17
|
};
|
|
16
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Whether a chunk has anything to forward besides PCM. CosyVoice3's native
|
|
21
|
+
* chunk streaming ends with an empty-PCM `isLast` chunk that carries the
|
|
22
|
+
* run's `speechTokenIds`; Audio8's native end markers are empty too but carry
|
|
23
|
+
* nothing, so the ops skip those and forward these.
|
|
24
|
+
*/
|
|
25
|
+
export function carriesSpeechTokenIds(chunk) {
|
|
26
|
+
return chunk.speechTokenIds !== undefined;
|
|
27
|
+
}
|
|
17
28
|
/**
|
|
18
29
|
* Appends a chunk's samples to `target` in place, which keeps collecting an
|
|
19
30
|
* utterance linear in its length — a natively chunked MOSS run emits dozens of
|
|
@@ -23,29 +34,116 @@ export function appendPcm(target, samples) {
|
|
|
23
34
|
for (let i = 0; i < samples.length; i++)
|
|
24
35
|
target.push(samples[i]);
|
|
25
36
|
}
|
|
26
|
-
|
|
27
|
-
|
|
37
|
+
/**
|
|
38
|
+
* One contiguous buffer from the engine's per-chunk output arrays: an
|
|
39
|
+
* `Int16Array` when every part is one (the engines' PCM), else a plain array.
|
|
40
|
+
*/
|
|
41
|
+
export function concatPcm(parts) {
|
|
42
|
+
if (parts.length === 1 && parts[0] instanceof Int16Array)
|
|
43
|
+
return parts[0];
|
|
44
|
+
if (!parts.every((p) => p instanceof Int16Array)) {
|
|
45
|
+
const out = [];
|
|
46
|
+
for (const p of parts)
|
|
47
|
+
appendPcm(out, p);
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
let total = 0;
|
|
51
|
+
for (const p of parts)
|
|
52
|
+
total += p.length;
|
|
53
|
+
const out = new Int16Array(total);
|
|
54
|
+
let offset = 0;
|
|
55
|
+
for (const p of parts) {
|
|
56
|
+
out.set(p, offset);
|
|
57
|
+
offset += p.length;
|
|
58
|
+
}
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The audio fields of a frame: int16 engine output rides the wire as base64
|
|
63
|
+
* `pcm`; anything else (test doubles, non-int16 engines) stays a `buffer`.
|
|
64
|
+
*/
|
|
65
|
+
export function pcmFrame(samples) {
|
|
66
|
+
if (samples instanceof Int16Array)
|
|
67
|
+
return { buffer: [], pcm: encodeTtsPcm(samples) };
|
|
68
|
+
return { buffer: Array.from(samples) };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The addon stats that keep their name on the wire. `audioDurationMs` is the
|
|
72
|
+
* one rename (`audioDuration`, the name the SDK has always exposed) and is
|
|
73
|
+
* handled on its own below.
|
|
74
|
+
*
|
|
75
|
+
* Two compile-time checks keep this table honest. `satisfies` requires every
|
|
76
|
+
* entry to be a key of both the addon's stats and `ttsStatsSchema`, and
|
|
77
|
+
* `UnforwardedTtsStats` below requires every key of either type to be an
|
|
78
|
+
* entry. A stat added to one side only, or to both without a line here, fails
|
|
79
|
+
* to compile rather than being dropped silently.
|
|
80
|
+
*/
|
|
81
|
+
const TTS_FORWARDED_STATS = [
|
|
82
|
+
'totalTime',
|
|
83
|
+
'realTimeFactor',
|
|
84
|
+
'tokensPerSecond',
|
|
85
|
+
'totalSamples',
|
|
86
|
+
'generatedFrames',
|
|
87
|
+
'firstAudioMs',
|
|
88
|
+
'backendDevice',
|
|
89
|
+
'backendId',
|
|
90
|
+
'gpuUnsupported',
|
|
91
|
+
'enhancerBackendDevice',
|
|
92
|
+
'enhancerBackendId',
|
|
93
|
+
'denoiserBackendDevice',
|
|
94
|
+
'denoiserBackendId',
|
|
95
|
+
'codecSidecarLoaded',
|
|
96
|
+
'codecOnCoreml',
|
|
97
|
+
// Parler Core ML sidecars
|
|
98
|
+
'decoderSidecarLoaded',
|
|
99
|
+
'dacSidecarLoaded',
|
|
100
|
+
'decoderOnCoreml',
|
|
101
|
+
'dacOnCoreml',
|
|
102
|
+
// Chatterbox
|
|
103
|
+
't3Ms',
|
|
104
|
+
's3genMs',
|
|
105
|
+
't3Tokens',
|
|
106
|
+
// CosyVoice3 / Audio8
|
|
107
|
+
'stageTotalMs',
|
|
108
|
+
// CosyVoice3
|
|
109
|
+
'lmPrefillMs',
|
|
110
|
+
'lmDecodeMs',
|
|
111
|
+
'flowFrontendMs',
|
|
112
|
+
'ditEulerMs',
|
|
113
|
+
'hiftF0Ms',
|
|
114
|
+
'hiftSourceMs',
|
|
115
|
+
'hiftStftMs',
|
|
116
|
+
'hiftDecodeMs',
|
|
117
|
+
'decodeSteps',
|
|
118
|
+
'speechTokens',
|
|
119
|
+
'flowFrames',
|
|
120
|
+
'melFrames',
|
|
121
|
+
'textIds',
|
|
122
|
+
'promptSpeechTokens',
|
|
123
|
+
// Audio8
|
|
124
|
+
'voiceEncodeMs',
|
|
125
|
+
'promptMs',
|
|
126
|
+
'prefillMs',
|
|
127
|
+
'sampleMs',
|
|
128
|
+
'fastDecodeMs',
|
|
129
|
+
'slowDecodeMs',
|
|
130
|
+
'codecLatentMs',
|
|
131
|
+
'codecSynthMs',
|
|
132
|
+
'resampleMs'
|
|
133
|
+
];
|
|
134
|
+
// Presence, not truthiness: `backendDevice: 0` (CPU) and a zero stage time are
|
|
135
|
+
// real reports, so only `undefined` is dropped.
|
|
28
136
|
export function collectTtsStats(response) {
|
|
29
137
|
const stats = response.stats;
|
|
30
138
|
if (!stats)
|
|
31
139
|
return {};
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
...(stats.gpuUnsupported !== undefined && { gpuUnsupported: stats.gpuUnsupported }),
|
|
42
|
-
...(stats.enhancerBackendDevice !== undefined && {
|
|
43
|
-
enhancerBackendDevice: stats.enhancerBackendDevice
|
|
44
|
-
}),
|
|
45
|
-
...(stats.enhancerBackendId !== undefined && { enhancerBackendId: stats.enhancerBackendId }),
|
|
46
|
-
...(stats.codecSidecarLoaded !== undefined && {
|
|
47
|
-
codecSidecarLoaded: stats.codecSidecarLoaded
|
|
48
|
-
}),
|
|
49
|
-
...(stats.codecOnCoreml !== undefined && { codecOnCoreml: stats.codecOnCoreml })
|
|
50
|
-
};
|
|
140
|
+
const out = {};
|
|
141
|
+
if (stats.audioDurationMs !== undefined)
|
|
142
|
+
out.audioDuration = stats.audioDurationMs;
|
|
143
|
+
for (const key of TTS_FORWARDED_STATS) {
|
|
144
|
+
const value = stats[key];
|
|
145
|
+
if (value !== undefined)
|
|
146
|
+
out[key] = value;
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
51
149
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/inference",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.0",
|
|
4
4
|
"description": "Bare-only in-process core of the QVAC SDK. Runs inference directly on the Bare runtime with explicit plugin assembly and no RPC, worker, or subprocess layer.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -82,9 +82,9 @@
|
|
|
82
82
|
"types": "./dist/models/registry/index.d.ts",
|
|
83
83
|
"import": "./dist/models/registry/index.js"
|
|
84
84
|
},
|
|
85
|
-
"./model-
|
|
86
|
-
"types": "./dist/
|
|
87
|
-
"import": "./dist/
|
|
85
|
+
"./model-cache-layout": {
|
|
86
|
+
"types": "./dist/model-cache-layout.d.ts",
|
|
87
|
+
"import": "./dist/model-cache-layout.js"
|
|
88
88
|
},
|
|
89
89
|
"./llamacpp-completion/plugin": {
|
|
90
90
|
"types": "./dist/plugins/builtin/llamacpp-completion/plugin.d.ts",
|
|
@@ -153,6 +153,10 @@
|
|
|
153
153
|
"./llamacpp-rerank/plugin": {
|
|
154
154
|
"types": "./dist/plugins/builtin/llamacpp-rerank/plugin.d.ts",
|
|
155
155
|
"import": "./dist/plugins/builtin/llamacpp-rerank/plugin.js"
|
|
156
|
+
},
|
|
157
|
+
"./ggml-image-embedding/plugin": {
|
|
158
|
+
"types": "./dist/plugins/builtin/ggml-image-embedding/plugin.d.ts",
|
|
159
|
+
"import": "./dist/plugins/builtin/ggml-image-embedding/plugin.js"
|
|
156
160
|
}
|
|
157
161
|
},
|
|
158
162
|
"files": [
|
|
@@ -168,26 +172,26 @@
|
|
|
168
172
|
"@qvac/error": "^0.1.1",
|
|
169
173
|
"@qvac/logging": "^0.1.1",
|
|
170
174
|
"@qvac/rag": "^0.9.0",
|
|
171
|
-
"@qvac/registry-client": "^0.
|
|
175
|
+
"@qvac/registry-client": "^0.8.0",
|
|
172
176
|
"bare-abort-controller": "^1.1.2",
|
|
173
177
|
"bare-buffer": "^3.6.2",
|
|
174
|
-
"bare-cpu-info": "0.1.1",
|
|
178
|
+
"bare-cpu-info": "^0.1.1",
|
|
175
179
|
"bare-crypto": "^1.15.3",
|
|
176
180
|
"bare-env": "^3.0.1",
|
|
177
|
-
"bare-fs": "^4.
|
|
178
|
-
"bare-gpu-info": "0.1.
|
|
181
|
+
"bare-fs": "^4.8.4",
|
|
182
|
+
"bare-gpu-info": "^0.1.3",
|
|
179
183
|
"bare-http1": "^4.6.2",
|
|
180
184
|
"bare-https": "^3.0.0",
|
|
181
185
|
"bare-net": "^2.3.2",
|
|
182
186
|
"bare-os": "^3.9.3",
|
|
183
187
|
"bare-path": "^3.1.1",
|
|
184
|
-
"bare-rpc": "^1.3.8",
|
|
185
188
|
"bare-stream": "^2.13.3",
|
|
186
189
|
"bare-url": "^2.4.5",
|
|
187
190
|
"bare-zlib": "^1.4.0",
|
|
188
191
|
"compact-encoding": "^3.3.0",
|
|
189
192
|
"corestore": "^7.11.0",
|
|
190
193
|
"fast-safe-stringify": "2.1.1",
|
|
194
|
+
"fs-native-extensions": "^1.5.0",
|
|
191
195
|
"hyperdrive": "^13.3.3",
|
|
192
196
|
"hyperswarm": "^4.17.0",
|
|
193
197
|
"semver": "^7.8.5",
|
|
@@ -196,21 +200,21 @@
|
|
|
196
200
|
"zod": "4.6.5"
|
|
197
201
|
},
|
|
198
202
|
"peerDependencies": {
|
|
199
|
-
"@qvac/asr-ggml": "^0.
|
|
200
|
-
"@qvac/audiogen-ggml": "^0.
|
|
201
|
-
"@qvac/bci-whispercpp": "^0.
|
|
202
|
-
"@qvac/classification-ggml": "^0.
|
|
203
|
-
"@qvac/decoder-audio": "^0.
|
|
204
|
-
"@qvac/diffusion-cpp": "^0.
|
|
205
|
-
"@qvac/embed-llamacpp": "^0.
|
|
203
|
+
"@qvac/asr-ggml": "^0.12.0",
|
|
204
|
+
"@qvac/audiogen-ggml": "^0.10.1",
|
|
205
|
+
"@qvac/bci-whispercpp": "^0.15.0",
|
|
206
|
+
"@qvac/classification-ggml": "^0.36.0",
|
|
207
|
+
"@qvac/decoder-audio": "^0.9.0",
|
|
208
|
+
"@qvac/diffusion-cpp": "^0.35.0",
|
|
209
|
+
"@qvac/embed-llamacpp": "^0.56.0",
|
|
206
210
|
"@qvac/langdetect-text": "^0.1.2",
|
|
207
|
-
"@qvac/llm-llamacpp": "^0.
|
|
208
|
-
"@qvac/ocr-ggml": "^0.
|
|
209
|
-
"@qvac/translation-nmtcpp": "^0.
|
|
210
|
-
"@qvac/tts-ggml": "^0.
|
|
211
|
-
"@qvac/turbovec-ggml": "^0.
|
|
212
|
-
"@qvac/vla-ggml": "^0.
|
|
213
|
-
"@qvac/ggml-rpc-server": "^0.
|
|
211
|
+
"@qvac/llm-llamacpp": "^0.70.0",
|
|
212
|
+
"@qvac/ocr-ggml": "^0.33.0",
|
|
213
|
+
"@qvac/translation-nmtcpp": "^0.25.0",
|
|
214
|
+
"@qvac/tts-ggml": "^0.15.0",
|
|
215
|
+
"@qvac/turbovec-ggml": "^0.5.0",
|
|
216
|
+
"@qvac/vla-ggml": "^0.36.0",
|
|
217
|
+
"@qvac/ggml-rpc-server": "^0.9.0"
|
|
214
218
|
},
|
|
215
219
|
"peerDependenciesMeta": {
|
|
216
220
|
"@qvac/asr-ggml": {
|
|
@@ -260,20 +264,20 @@
|
|
|
260
264
|
}
|
|
261
265
|
},
|
|
262
266
|
"devDependencies": {
|
|
263
|
-
"@qvac/asr-ggml": "^0.
|
|
264
|
-
"@qvac/audiogen-ggml": "^0.
|
|
265
|
-
"@qvac/bci-whispercpp": "^0.
|
|
266
|
-
"@qvac/classification-ggml": "^0.
|
|
267
|
-
"@qvac/decoder-audio": "^0.
|
|
268
|
-
"@qvac/diffusion-cpp": "^0.
|
|
269
|
-
"@qvac/embed-llamacpp": "^0.
|
|
267
|
+
"@qvac/asr-ggml": "^0.12.0",
|
|
268
|
+
"@qvac/audiogen-ggml": "^0.10.1",
|
|
269
|
+
"@qvac/bci-whispercpp": "^0.15.0",
|
|
270
|
+
"@qvac/classification-ggml": "^0.36.0",
|
|
271
|
+
"@qvac/decoder-audio": "^0.9.0",
|
|
272
|
+
"@qvac/diffusion-cpp": "^0.35.0",
|
|
273
|
+
"@qvac/embed-llamacpp": "^0.56.0",
|
|
270
274
|
"@qvac/langdetect-text": "^0.1.2",
|
|
271
|
-
"@qvac/llm-llamacpp": "^0.
|
|
272
|
-
"@qvac/ocr-ggml": "^0.
|
|
273
|
-
"@qvac/translation-nmtcpp": "^0.
|
|
274
|
-
"@qvac/tts-ggml": "^0.
|
|
275
|
-
"@qvac/turbovec-ggml": "^0.
|
|
276
|
-
"@qvac/vla-ggml": "^0.
|
|
275
|
+
"@qvac/llm-llamacpp": "^0.70.0",
|
|
276
|
+
"@qvac/ocr-ggml": "^0.33.0",
|
|
277
|
+
"@qvac/translation-nmtcpp": "^0.25.0",
|
|
278
|
+
"@qvac/tts-ggml": "^0.15.0",
|
|
279
|
+
"@qvac/turbovec-ggml": "^0.5.0",
|
|
280
|
+
"@qvac/vla-ggml": "^0.36.0",
|
|
277
281
|
"@types/brittle": "^3.5.0",
|
|
278
282
|
"bare": "1.34.1 || >=1.35.1",
|
|
279
283
|
"bare-console": "*",
|
|
@@ -288,9 +292,9 @@
|
|
|
288
292
|
"prettier-config-holepunch": "^2.0.0",
|
|
289
293
|
"tsc-alias": "^1.8.17",
|
|
290
294
|
"typescript": "^5.9.3",
|
|
291
|
-
"@qvac/ggml-rpc-server": "^0.
|
|
295
|
+
"@qvac/ggml-rpc-server": "^0.9.0"
|
|
292
296
|
},
|
|
293
|
-
"gitHead": "
|
|
297
|
+
"gitHead": "1f559be4e208ee7e58e775bde4dd078263ca740c",
|
|
294
298
|
"scripts": {
|
|
295
299
|
"lint": "lunte",
|
|
296
300
|
"lint:fix": "lunte --fix",
|
|
@@ -304,6 +308,6 @@
|
|
|
304
308
|
"test": "npm run build:test && brittle-bare 'test/dist/test/**/*.test.js'",
|
|
305
309
|
"test:bare": "npm run test",
|
|
306
310
|
"check-models": "bare ./dist/models/update-models/index.js --check",
|
|
307
|
-
"update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts
|
|
311
|
+
"update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts"
|
|
308
312
|
}
|
|
309
313
|
}
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { NativeProbeFit } from '../schemas/index.js';
|
|
2
|
-
import type { FitProbeResult } from '../model-fit/run/engine.js';
|
|
3
|
-
export declare function classifyFit(probe: FitProbeResult, provenance: {
|
|
4
|
-
basis: 'native-probe';
|
|
5
|
-
estimatorVersion: string;
|
|
6
|
-
}): NativeProbeFit;
|
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
/** The llama fitters append a host row after the devices they assigned to. */
|
|
2
|
-
const HOST_ROW = 'host';
|
|
3
|
-
/** ggml's own name for the CPU backend device, which the fitter may assign to. */
|
|
4
|
-
const CPU_DEVICE = 'CPU';
|
|
5
|
-
function isLlama(probe) {
|
|
6
|
-
return probe.engine === 'llm-llamacpp' || probe.engine === 'embed-llamacpp';
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* `error` is the engine reporting it could not read the model or find a backend
|
|
10
|
-
* device: absence of evidence, not evidence of insufficiency.
|
|
11
|
-
*/
|
|
12
|
-
function verdictFor(status) {
|
|
13
|
-
if (status === 'fits')
|
|
14
|
-
return 'fit';
|
|
15
|
-
if (status === 'does-not-fit')
|
|
16
|
-
return 'does-not-fit';
|
|
17
|
-
return 'unknown';
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* The diffusion fitter reports a placement alongside its reason. `changed`
|
|
21
|
-
* means it reached a fitting arrangement only by moving modules between
|
|
22
|
-
* backends, which is why the status is already `does-not-fit`.
|
|
23
|
-
*/
|
|
24
|
-
function diffusionReason(result) {
|
|
25
|
-
if (result.status === 'fits')
|
|
26
|
-
return 'fits';
|
|
27
|
-
if (result.status === 'error')
|
|
28
|
-
return result.reason;
|
|
29
|
-
return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
|
|
30
|
-
}
|
|
31
|
-
function reasonFor(probe) {
|
|
32
|
-
return probe.engine === 'diffusion-cpp' ? diffusionReason(probe.result) : probe.result.reason;
|
|
33
|
-
}
|
|
34
|
-
function sumOf(...values) {
|
|
35
|
-
const present = values.filter((value) => value !== undefined);
|
|
36
|
-
return present.length === 0 ? undefined : present.reduce((total, value) => total + value, 0);
|
|
37
|
-
}
|
|
38
|
-
function breakdown(weights, context, compute) {
|
|
39
|
-
return {
|
|
40
|
-
...(weights !== undefined && { weightsBytes: weights }),
|
|
41
|
-
...(context !== undefined && { contextBytes: context }),
|
|
42
|
-
...(compute !== undefined && { computeBytes: compute })
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* The parts of `deviceBytes` the engine separates. Audiogen reports a peak
|
|
47
|
-
* across pipeline phases and diffusion a per-module table, neither of which
|
|
48
|
-
* divides into these three, so both come back empty. OCR sizes its weights
|
|
49
|
-
* alone and leaves the image-dependent memory uncounted.
|
|
50
|
-
*
|
|
51
|
-
* The llama rows are summed over the same devices the engine's own
|
|
52
|
-
* `deviceBytes` covers, which is every row but `host`.
|
|
53
|
-
*/
|
|
54
|
-
function breakdownFor(probe) {
|
|
55
|
-
if (isLlama(probe)) {
|
|
56
|
-
const devices = probe.result.devices.filter((device) => device.name !== HOST_ROW);
|
|
57
|
-
return breakdown(sumOf(...devices.map((device) => device.modelBytes)), sumOf(...devices.map((device) => device.contextBytes)), sumOf(...devices.map((device) => device.computeBytes)));
|
|
58
|
-
}
|
|
59
|
-
// Whisper names its parts `kvBytes` and `computeBytes`, parakeet splits
|
|
60
|
-
// compute across encoder and decoder; an engine fills one set or the other.
|
|
61
|
-
if (probe.engine === 'asr-ggml') {
|
|
62
|
-
const result = probe.result;
|
|
63
|
-
// A VAD model's weights are inside `deviceBytes`, so they belong in the
|
|
64
|
-
// weights the breakdown divides it into.
|
|
65
|
-
return breakdown(sumOf(result.weightsBytes, result.vadBytes), sumOf(result.kvBytes, result.decoderStateBytes), sumOf(result.computeBytes, result.encoderComputeBytes, result.decoderComputeBytes));
|
|
66
|
-
}
|
|
67
|
-
if (probe.engine === 'bci-whispercpp') {
|
|
68
|
-
const result = probe.result;
|
|
69
|
-
return breakdown(result.weightsBytes, result.kvBytes, result.computeBytes);
|
|
70
|
-
}
|
|
71
|
-
if (probe.engine === 'tts-ggml') {
|
|
72
|
-
const result = probe.result;
|
|
73
|
-
return breakdown(result.weightsBytes, result.stateBytes, sumOf(result.lmComputeBytes, result.codecComputeBytes));
|
|
74
|
-
}
|
|
75
|
-
if (probe.engine === 'ocr-ggml') {
|
|
76
|
-
return breakdown(probe.result.weightsBytes, undefined, undefined);
|
|
77
|
-
}
|
|
78
|
-
return {};
|
|
79
|
-
}
|
|
80
|
-
function llamaProjection(probe) {
|
|
81
|
-
const first = probe.result.devices.find((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE);
|
|
82
|
-
const devices = probe.result.devices
|
|
83
|
-
.filter((device) => device.name !== HOST_ROW)
|
|
84
|
-
.map((device) => ({
|
|
85
|
-
name: device.name,
|
|
86
|
-
totalBytes: device.totalBytes,
|
|
87
|
-
freeBytes: device.freeBytes,
|
|
88
|
-
weightsBytes: device.modelBytes,
|
|
89
|
-
contextBytes: device.contextBytes,
|
|
90
|
-
computeBytes: device.computeBytes
|
|
91
|
-
}));
|
|
92
|
-
return {
|
|
93
|
-
deviceBytes: probe.result.deviceBytes,
|
|
94
|
-
hostBytes: probe.result.hostBytes,
|
|
95
|
-
...breakdownFor(probe),
|
|
96
|
-
...(devices.length > 0 && { devices }),
|
|
97
|
-
...(first !== undefined && {
|
|
98
|
-
deviceName: first.name,
|
|
99
|
-
deviceFreeBytes: first.freeBytes,
|
|
100
|
-
deviceTotalBytes: first.totalBytes
|
|
101
|
-
})
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* The translation fitter reports one allowance against the free memory of the
|
|
106
|
-
* device it ran on. Only the file bytes inside it are weights; the rest does
|
|
107
|
-
* not divide into context and compute.
|
|
108
|
-
*/
|
|
109
|
-
function nmtProjection(result) {
|
|
110
|
-
return {
|
|
111
|
-
deviceName: result.backend,
|
|
112
|
-
deviceBytes: result.requiredBytes,
|
|
113
|
-
deviceFreeBytes: result.freeBytes,
|
|
114
|
-
weightsBytes: result.modelBytes,
|
|
115
|
-
report: result.report
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
function projectionFor(probe) {
|
|
119
|
-
if (probe.engine === 'diffusion-cpp') {
|
|
120
|
-
return { deviceName: probe.result.backend, report: probe.result.report };
|
|
121
|
-
}
|
|
122
|
-
if (probe.engine === 'translation-nmtcpp')
|
|
123
|
-
return nmtProjection(probe.result);
|
|
124
|
-
if (isLlama(probe))
|
|
125
|
-
return llamaProjection(probe);
|
|
126
|
-
const result = probe.result;
|
|
127
|
-
return {
|
|
128
|
-
deviceName: result.deviceName,
|
|
129
|
-
deviceBytes: result.deviceBytes,
|
|
130
|
-
hostBytes: result.hostBytes,
|
|
131
|
-
deviceFreeBytes: result.deviceFreeBytes,
|
|
132
|
-
deviceTotalBytes: result.deviceTotalBytes,
|
|
133
|
-
report: result.report,
|
|
134
|
-
...breakdownFor(probe)
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
/**
|
|
138
|
-
* Only the llama fitters resolve a placement; the other engines' loads are not
|
|
139
|
-
* layer-sliced across devices.
|
|
140
|
-
*
|
|
141
|
-
* `nGpuDevices` counts the devices the weights were actually placed on, less
|
|
142
|
-
* the CPU backend device, which the fitter assigns to like any other but whose
|
|
143
|
-
* demand is host memory.
|
|
144
|
-
*/
|
|
145
|
-
function planFor(probe) {
|
|
146
|
-
if (!isLlama(probe))
|
|
147
|
-
return undefined;
|
|
148
|
-
const assigned = probe.result.devices.filter((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE && device.modelBytes > 0);
|
|
149
|
-
return {
|
|
150
|
-
nCtx: probe.result.ctxSize,
|
|
151
|
-
nGpuLayers: probe.result.gpuLayers,
|
|
152
|
-
nGpuDevices: assigned.length
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
export function classifyFit(probe, provenance) {
|
|
156
|
-
const verdict = verdictFor(probe.result.status);
|
|
157
|
-
const plan = verdict === 'fit' ? planFor(probe) : undefined;
|
|
158
|
-
return {
|
|
159
|
-
...provenance,
|
|
160
|
-
engine: probe.engine,
|
|
161
|
-
verdict,
|
|
162
|
-
reason: reasonFor(probe),
|
|
163
|
-
...(plan !== undefined && { plan }),
|
|
164
|
-
...(verdict !== 'unknown' && { projection: projectionFor(probe) })
|
|
165
|
-
};
|
|
166
|
-
}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import { type LoadDescription } from '../../model-fit/description/resolve-load.js';
|
|
2
|
-
import type { FitStubOptions, FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
|
|
3
|
-
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../../model-fit/load-check.js';
|
|
4
|
-
export type LoadFitOutcome = {
|
|
5
|
-
status: 'projected';
|
|
6
|
-
fit: AdvisoryFitOutcome;
|
|
7
|
-
} | {
|
|
8
|
-
status: 'no-stub';
|
|
9
|
-
reason: FitStubUnavailableReason;
|
|
10
|
-
message?: string;
|
|
11
|
-
} | {
|
|
12
|
-
status: 'unsupported-load';
|
|
13
|
-
detail: string;
|
|
14
|
-
};
|
|
15
|
-
export interface LoadFitOptions {
|
|
16
|
-
stub?: FitStubOptions;
|
|
17
|
-
fit?: AdvisoryFitOptions;
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* Projects the load a caller describes, before its weights exist locally.
|
|
21
|
-
*
|
|
22
|
-
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
23
|
-
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
24
|
-
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
25
|
-
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
26
|
-
*/
|
|
27
|
-
export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import { resolveLoadFromStubs } from '../../model-fit/description/resolve-load.js';
|
|
2
|
-
import { runAdvisoryFitCheck } from '../../model-fit/load-check.js';
|
|
3
|
-
/**
|
|
4
|
-
* Projects the load a caller describes, before its weights exist locally.
|
|
5
|
-
*
|
|
6
|
-
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
7
|
-
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
8
|
-
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
9
|
-
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
10
|
-
*/
|
|
11
|
-
export async function projectFitFromLoad(load, modelId, options = {}) {
|
|
12
|
-
const resolved = await resolveLoadFromStubs(load, options.stub);
|
|
13
|
-
if (resolved.status !== 'resolved')
|
|
14
|
-
return resolved;
|
|
15
|
-
try {
|
|
16
|
-
const fit = await runAdvisoryFitCheck({
|
|
17
|
-
modelId,
|
|
18
|
-
modelType: load.modelType,
|
|
19
|
-
modelPath: resolved.modelPath,
|
|
20
|
-
modelConfig: resolved.modelConfig,
|
|
21
|
-
transcriptionWorkload: load.transcriptionWorkload,
|
|
22
|
-
ttsWorkload: load.ttsWorkload,
|
|
23
|
-
artifacts: resolved.artifacts,
|
|
24
|
-
// Nothing here assembles a shard set, so no load may claim the fitter
|
|
25
|
-
// is looking at a split model.
|
|
26
|
-
isShardedModel: false
|
|
27
|
-
}, options.fit);
|
|
28
|
-
return { status: 'projected', fit };
|
|
29
|
-
}
|
|
30
|
-
finally {
|
|
31
|
-
await resolved.release();
|
|
32
|
-
}
|
|
33
|
-
}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { ModelFitWorkload } from '../schemas/assess-model-fit.js';
|
|
2
|
-
import type { ModelResourceProfile } from '../schemas/model-resource-profile.js';
|
|
3
|
-
export declare const FLOOR_VERSION = "floor-v1";
|
|
4
|
-
export interface FloorInput {
|
|
5
|
-
profile: ModelResourceProfile;
|
|
6
|
-
workload: ModelFitWorkload;
|
|
7
|
-
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
8
|
-
extraArtifactBytes: number;
|
|
9
|
-
}
|
|
10
|
-
/** A lower bound with no upper bound, and how it was assembled. */
|
|
11
|
-
export interface FloorResult {
|
|
12
|
-
bytes: number;
|
|
13
|
-
reasons: readonly string[];
|
|
14
|
-
assumptions: readonly string[];
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* The smallest resident footprint the catalog facts prove: artifact bytes, plus
|
|
18
|
-
* the KV cache for a llama.cpp model at the cheapest element width the engine
|
|
19
|
-
* can default to.
|
|
20
|
-
*
|
|
21
|
-
* Every term the real load adds on top — weight slack, engine overhead, compute
|
|
22
|
-
* buffers, a completion's working peak — is non-negative, so a floor with all of
|
|
23
|
-
* them at zero is unconditionally valid on any platform and any backend. It can
|
|
24
|
-
* therefore refuse a model (`likely-too-large` when even the floor is over
|
|
25
|
-
* budget) but never confirm one, since how far above the floor the real load
|
|
26
|
-
* lands is unmeasured here.
|
|
27
|
-
*
|
|
28
|
-
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
29
|
-
* fallback wherever no engine measurement is available.
|
|
30
|
-
*/
|
|
31
|
-
export declare function computeFloor(input: FloorInput): FloorResult;
|
package/dist/model-fit/floor.js
DELETED
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from '../model-fit/kv-cache.js';
|
|
2
|
-
export const FLOOR_VERSION = 'floor-v1';
|
|
3
|
-
/**
|
|
4
|
-
* Engines whose load builds a KV cache the file sizes exactly, so the floor can
|
|
5
|
-
* count it. Every other engine's working memory is engine-owned and unsized
|
|
6
|
-
* here; the floor for those is the weights alone.
|
|
7
|
-
*/
|
|
8
|
-
const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
|
|
9
|
-
/**
|
|
10
|
-
* The smallest resident footprint the catalog facts prove: artifact bytes, plus
|
|
11
|
-
* the KV cache for a llama.cpp model at the cheapest element width the engine
|
|
12
|
-
* can default to.
|
|
13
|
-
*
|
|
14
|
-
* Every term the real load adds on top — weight slack, engine overhead, compute
|
|
15
|
-
* buffers, a completion's working peak — is non-negative, so a floor with all of
|
|
16
|
-
* them at zero is unconditionally valid on any platform and any backend. It can
|
|
17
|
-
* therefore refuse a model (`likely-too-large` when even the floor is over
|
|
18
|
-
* budget) but never confirm one, since how far above the floor the real load
|
|
19
|
-
* lands is unmeasured here.
|
|
20
|
-
*
|
|
21
|
-
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
22
|
-
* fallback wherever no engine measurement is available.
|
|
23
|
-
*/
|
|
24
|
-
export function computeFloor(input) {
|
|
25
|
-
const { profile, workload, extraArtifactBytes } = input;
|
|
26
|
-
const reasons = [];
|
|
27
|
-
const assumptions = [
|
|
28
|
-
KV_CACHE_ENGINES.has(profile.engine)
|
|
29
|
-
? LLAMA_WEIGHTS_ASSUMPTION
|
|
30
|
-
: 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
|
|
31
|
-
'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
|
|
32
|
-
];
|
|
33
|
-
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
34
|
-
if (extraArtifactBytes > 0) {
|
|
35
|
-
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
36
|
-
}
|
|
37
|
-
if (!KV_CACHE_ENGINES.has(profile.engine)) {
|
|
38
|
-
reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
|
|
39
|
-
return { bytes: artifactBytes, reasons, assumptions };
|
|
40
|
-
}
|
|
41
|
-
if (workload.kind !== 'llm') {
|
|
42
|
-
reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
|
|
43
|
-
return { bytes: artifactBytes, reasons, assumptions };
|
|
44
|
-
}
|
|
45
|
-
const facts = profile.ggufFacts;
|
|
46
|
-
if (!facts) {
|
|
47
|
-
reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
|
|
48
|
-
return { bytes: artifactBytes, reasons, assumptions };
|
|
49
|
-
}
|
|
50
|
-
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
51
|
-
if (contextTokens > facts.contextLength) {
|
|
52
|
-
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
53
|
-
contextTokens = facts.contextLength;
|
|
54
|
-
}
|
|
55
|
-
// The narrowest default the engine can pick on any backend, so the floor
|
|
56
|
-
// holds whether the model lands on a GPU (q8_0 with flash attention) or on
|
|
57
|
-
// the CPU (f16). Architectures that disable flash attention stay f16 anyway.
|
|
58
|
-
const element = narrowestKvElementBytes(facts);
|
|
59
|
-
const kv = kvCacheFloorBytes(facts, contextTokens, element);
|
|
60
|
-
assumptions.push(element < 2
|
|
61
|
-
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
|
-
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
63
|
-
reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
|
|
64
|
-
return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
|
|
65
|
-
}
|