@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import type { GgufFacts } from '../schemas/model-resource-profile.js';
|
|
2
|
-
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
3
|
-
/**
|
|
4
|
-
* The narrowest KV-cache element width the engine can default to for this
|
|
5
|
-
* model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
|
|
6
|
-
*/
|
|
7
|
-
export declare function narrowestKvElementBytes(facts: GgufFacts): number;
|
|
8
|
-
/**
|
|
9
|
-
* The smallest KV cache the requested context can need, at a fixed element
|
|
10
|
-
* width and clamped to the trained context.
|
|
11
|
-
*
|
|
12
|
-
* Three cases, in order of how much the file actually tells us:
|
|
13
|
-
*
|
|
14
|
-
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
15
|
-
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
16
|
-
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
17
|
-
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
18
|
-
* blocks are which is engine-owned, so the fewest full blocks are counted.
|
|
19
|
-
* 3. **Flat** — every block holds the same cache. A declared sliding window
|
|
20
|
-
* with no per-layer pattern is engine-owned, so every block is counted
|
|
21
|
-
* windowed.
|
|
22
|
-
*/
|
|
23
|
-
export declare function kvCacheFloorBytes(facts: GgufFacts, contextTokens: number, elementBytes: number): number;
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* KV-cache element widths, in bytes per element.
|
|
3
|
-
*
|
|
4
|
-
* `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
|
|
5
|
-
* on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
|
|
6
|
-
* instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318).
|
|
7
|
-
* `q8_0` packs 32 elements into a 34-byte block.
|
|
8
|
-
*/
|
|
9
|
-
const F16_BYTES_PER_ELEMENT = 2;
|
|
10
|
-
const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
|
|
11
|
-
/** Recurrent/SSM state is kept in f32. */
|
|
12
|
-
const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
13
|
-
/**
|
|
14
|
-
* Architectures that disable flash attention, so the GPU `q8_0` KV default
|
|
15
|
-
* never applies and the cache stays `f16` on every backend.
|
|
16
|
-
*/
|
|
17
|
-
function disablesFlashAttention(architecture) {
|
|
18
|
-
return architecture.startsWith('bitnet');
|
|
19
|
-
}
|
|
20
|
-
// Shared with the computed floor so a mixed set carries one weights assumption, not two.
|
|
21
|
-
export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
|
|
22
|
-
/**
|
|
23
|
-
* The narrowest KV-cache element width the engine can default to for this
|
|
24
|
-
* model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
|
|
25
|
-
*/
|
|
26
|
-
export function narrowestKvElementBytes(facts) {
|
|
27
|
-
return disablesFlashAttention(facts.architecture) ? F16_BYTES_PER_ELEMENT : Q8_0_BYTES_PER_ELEMENT;
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* The smallest KV cache the requested context can need, at a fixed element
|
|
31
|
-
* width and clamped to the trained context.
|
|
32
|
-
*
|
|
33
|
-
* Three cases, in order of how much the file actually tells us:
|
|
34
|
-
*
|
|
35
|
-
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
36
|
-
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
37
|
-
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
38
|
-
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
39
|
-
* blocks are which is engine-owned, so the fewest full blocks are counted.
|
|
40
|
-
* 3. **Flat** — every block holds the same cache. A declared sliding window
|
|
41
|
-
* with no per-layer pattern is engine-owned, so every block is counted
|
|
42
|
-
* windowed.
|
|
43
|
-
*/
|
|
44
|
-
export function kvCacheFloorBytes(facts, contextTokens, elementBytes) {
|
|
45
|
-
const tokens = Math.min(contextTokens, facts.contextLength);
|
|
46
|
-
if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
|
|
47
|
-
return layerClassBytes(facts.kvLayerClasses, facts, tokens, elementBytes);
|
|
48
|
-
}
|
|
49
|
-
const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
|
|
50
|
-
if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
|
|
51
|
-
const fullBlocks = Math.floor(facts.blockCount / facts.fullAttentionInterval);
|
|
52
|
-
const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocks);
|
|
53
|
-
return fullBlocks * perBlockPerToken * tokens * elementBytes + ssm;
|
|
54
|
-
}
|
|
55
|
-
if (facts.slidingWindow) {
|
|
56
|
-
const windowedTokens = Math.min(tokens, facts.slidingWindow);
|
|
57
|
-
return facts.blockCount * perBlockPerToken * windowedTokens * elementBytes;
|
|
58
|
-
}
|
|
59
|
-
return facts.blockCount * perBlockPerToken * tokens * elementBytes;
|
|
60
|
-
}
|
|
61
|
-
function layerClassBytes(classes, facts, contextTokens, elementBytes) {
|
|
62
|
-
let total = 0;
|
|
63
|
-
for (const layerClass of classes) {
|
|
64
|
-
const tokens = layerClass.windowed && facts.slidingWindow
|
|
65
|
-
? Math.min(contextTokens, facts.slidingWindow)
|
|
66
|
-
: contextTokens;
|
|
67
|
-
total +=
|
|
68
|
-
layerClass.count *
|
|
69
|
-
layerClass.headCountKv *
|
|
70
|
-
(layerClass.keyLength + layerClass.valueLength) *
|
|
71
|
-
tokens *
|
|
72
|
-
elementBytes;
|
|
73
|
-
}
|
|
74
|
-
return total;
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
|
|
78
|
-
* the SSM state plus its convolution window, per block.
|
|
79
|
-
*/
|
|
80
|
-
function ssmStateBytes(facts, recurrentBlocks) {
|
|
81
|
-
if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
|
|
82
|
-
return 0;
|
|
83
|
-
const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
|
|
84
|
-
return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
|
|
85
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Platforms this assessment covers. A platform absent from this list assesses
|
|
3
|
-
* as `unknown`, because nothing fixes the memory budget it would be judged
|
|
4
|
-
* against.
|
|
5
|
-
*/
|
|
6
|
-
export type ModelFitPlatform = 'darwin-arm64' | 'darwin-x64' | 'linux-arm64' | 'linux-x64' | 'win32-x64' | 'win32-arm64' | 'android-arm64' | 'ios-arm64';
|
|
7
|
-
/**
|
|
8
|
-
* Maps the runtime's platform and architecture onto a platform this assessment
|
|
9
|
-
* covers.
|
|
10
|
-
*
|
|
11
|
-
* @returns `undefined` for any pair outside that set, which assesses as
|
|
12
|
-
* `unknown`.
|
|
13
|
-
*/
|
|
14
|
-
export declare function detectPlatform(): ModelFitPlatform | undefined;
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import os from 'bare-os';
|
|
2
|
-
/**
|
|
3
|
-
* Maps the runtime's platform and architecture onto a platform this assessment
|
|
4
|
-
* covers.
|
|
5
|
-
*
|
|
6
|
-
* @returns `undefined` for any pair outside that set, which assesses as
|
|
7
|
-
* `unknown`.
|
|
8
|
-
*/
|
|
9
|
-
export function detectPlatform() {
|
|
10
|
-
const key = `${os.platform()}-${os.arch()}`;
|
|
11
|
-
switch (key) {
|
|
12
|
-
case 'darwin-arm64':
|
|
13
|
-
case 'darwin-x64':
|
|
14
|
-
case 'linux-arm64':
|
|
15
|
-
case 'linux-x64':
|
|
16
|
-
case 'win32-x64':
|
|
17
|
-
// No engine addon is built for win32-arm64 (`@qvac/llm-llamacpp/prebuilds`
|
|
18
|
-
// has no such target), so no load on it reaches a fitter.
|
|
19
|
-
case 'win32-arm64':
|
|
20
|
-
case 'android-arm64':
|
|
21
|
-
case 'ios-arm64':
|
|
22
|
-
return key;
|
|
23
|
-
default:
|
|
24
|
-
return undefined;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { TtsFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
-
import { type CanonicalModelType, type TranscriptionFitWorkload } from '../../schemas/index.js';
|
|
3
|
-
import type { FitProbeRequest } from '../../model-fit/run/engine.js';
|
|
4
|
-
export type FitRequestPlan = {
|
|
5
|
-
supported: true;
|
|
6
|
-
probe: FitProbeRequest;
|
|
7
|
-
} | {
|
|
8
|
-
supported: false;
|
|
9
|
-
detail: string;
|
|
10
|
-
};
|
|
11
|
-
export interface CreateFitRequestParams {
|
|
12
|
-
ttsWorkload?: TtsFitWorkload | undefined;
|
|
13
|
-
modelType: CanonicalModelType;
|
|
14
|
-
modelPath: string;
|
|
15
|
-
modelConfig: unknown;
|
|
16
|
-
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
17
|
-
artifacts?: Record<string, string> | undefined;
|
|
18
|
-
isShardedModel: boolean;
|
|
19
|
-
/** Free memory that must remain for the projection to count as fitting. */
|
|
20
|
-
marginBytes?: number | undefined;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Builds a fit request from the same resolved config and artifacts the real
|
|
24
|
-
* load is about to use, or explains why this load cannot be answered for.
|
|
25
|
-
*
|
|
26
|
-
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
27
|
-
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
28
|
-
* — are refused here so no engine is asked a question it cannot answer.
|
|
29
|
-
* Value-level policy stays inside each engine's own fitter.
|
|
30
|
-
*/
|
|
31
|
-
export declare function createFitRequest(params: CreateFitRequestParams): FitRequestPlan;
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { ModelType } from '../../schemas/index.js';
|
|
2
|
-
import { createBciFitRequest, createMossTranscribeFitRequest, createParakeetFitRequest, createWhisperFitRequest } from '../../model-fit/request/engines/asr.js';
|
|
3
|
-
import { createAudiogenFitRequest } from '../../model-fit/request/engines/audiogen.js';
|
|
4
|
-
import { createDiffusionFitRequest } from '../../model-fit/request/engines/diffusion.js';
|
|
5
|
-
import { createLlamaFitRequest, llamaLoadKindFor } from '../../model-fit/request/engines/llama.js';
|
|
6
|
-
import { createNmtFitRequest } from '../../model-fit/request/engines/nmt.js';
|
|
7
|
-
import { createOcrFitRequest } from '../../model-fit/request/engines/ocr.js';
|
|
8
|
-
import { createTtsFitRequest } from '../../model-fit/request/engines/tts.js';
|
|
9
|
-
/**
|
|
10
|
-
* Builds a fit request from the same resolved config and artifacts the real
|
|
11
|
-
* load is about to use, or explains why this load cannot be answered for.
|
|
12
|
-
*
|
|
13
|
-
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
14
|
-
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
15
|
-
* — are refused here so no engine is asked a question it cannot answer.
|
|
16
|
-
* Value-level policy stays inside each engine's own fitter.
|
|
17
|
-
*/
|
|
18
|
-
export function createFitRequest(params) {
|
|
19
|
-
const { modelType, modelPath, modelConfig, artifacts, marginBytes } = params;
|
|
20
|
-
if (params.transcriptionWorkload !== undefined && modelType !== ModelType.mossTranscribe) {
|
|
21
|
-
return {
|
|
22
|
-
supported: false,
|
|
23
|
-
detail: 'transcriptionWorkload is supported only for MOSS transcription'
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
// Every engine reads whole files rather than a shard set, so a split model is
|
|
27
|
-
// not a shape any of them can be pointed at.
|
|
28
|
-
if (params.isShardedModel) {
|
|
29
|
-
return { supported: false, detail: 'sharded models are not representable' };
|
|
30
|
-
}
|
|
31
|
-
const loadKind = llamaLoadKindFor(modelType);
|
|
32
|
-
if (loadKind !== undefined) {
|
|
33
|
-
return createLlamaFitRequest({ loadKind, modelPath, modelConfig, artifacts, marginBytes });
|
|
34
|
-
}
|
|
35
|
-
const common = { modelPath, modelConfig, artifacts, marginBytes };
|
|
36
|
-
switch (modelType) {
|
|
37
|
-
case ModelType.mossTranscribe:
|
|
38
|
-
return createMossTranscribeFitRequest({
|
|
39
|
-
...common,
|
|
40
|
-
transcriptionWorkload: params.transcriptionWorkload
|
|
41
|
-
});
|
|
42
|
-
case ModelType.whispercppTranscription:
|
|
43
|
-
return createWhisperFitRequest(common);
|
|
44
|
-
case ModelType.parakeetTranscription:
|
|
45
|
-
return createParakeetFitRequest(common);
|
|
46
|
-
case ModelType.bciWhispercppTranscription:
|
|
47
|
-
return createBciFitRequest(common);
|
|
48
|
-
case ModelType.ttsGgml:
|
|
49
|
-
return createTtsFitRequest({ ...common, ttsWorkload: params.ttsWorkload });
|
|
50
|
-
case ModelType.audiogenGgml:
|
|
51
|
-
return createAudiogenFitRequest(common);
|
|
52
|
-
case ModelType.sdcppGeneration:
|
|
53
|
-
return createDiffusionFitRequest({ modelPath, modelConfig, artifacts });
|
|
54
|
-
case ModelType.ggmlOcr:
|
|
55
|
-
return createOcrFitRequest(common);
|
|
56
|
-
case ModelType.nmtcppTranslation:
|
|
57
|
-
return createNmtFitRequest(common);
|
|
58
|
-
default:
|
|
59
|
-
return { supported: false, detail: `no fitter for model type: ${modelType}` };
|
|
60
|
-
}
|
|
61
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { TranscriptionFitWorkload } from '../../../schemas/index.js';
|
|
2
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
3
|
-
export interface AsrFitRequestParams {
|
|
4
|
-
modelPath: string;
|
|
5
|
-
modelConfig: unknown;
|
|
6
|
-
artifacts?: Record<string, string> | undefined;
|
|
7
|
-
marginBytes?: number | undefined;
|
|
8
|
-
}
|
|
9
|
-
export declare function createMossTranscribeFitRequest(params: AsrFitRequestParams & {
|
|
10
|
-
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
11
|
-
}): FitRequestPlan;
|
|
12
|
-
export declare function createWhisperFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
13
|
-
export declare function createParakeetFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
14
|
-
export declare function createBciFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
|
-
export interface AudiogenFitRequestParams {
|
|
3
|
-
modelPath: string;
|
|
4
|
-
modelConfig: unknown;
|
|
5
|
-
artifacts?: Record<string, string> | undefined;
|
|
6
|
-
marginBytes?: number | undefined;
|
|
7
|
-
}
|
|
8
|
-
export declare function createAudiogenFitRequest(params: AudiogenFitRequestParams): FitRequestPlan;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
|
-
export interface DiffusionFitRequestParams {
|
|
3
|
-
modelPath: string;
|
|
4
|
-
modelConfig: unknown;
|
|
5
|
-
artifacts?: Record<string, string> | undefined;
|
|
6
|
-
}
|
|
7
|
-
export declare function createDiffusionFitRequest(params: DiffusionFitRequestParams): FitRequestPlan;
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import { flattenUpscalerKeys, toEsrganAddonConfig } from '../../../plugins/builtin/sdcpp-generation/upscaler-config.js';
|
|
2
|
-
import { splitWorldConfig } from '../../../plugins/builtin/sdcpp-generation/world-config.js';
|
|
3
|
-
/**
|
|
4
|
-
* Frames to project a video load at. The fitter reads one frame when nothing
|
|
5
|
-
* is passed, which projects image generation; `run` generates 33. The frame
|
|
6
|
-
* count is a generation parameter, and this assessment sees load config alone.
|
|
7
|
-
*/
|
|
8
|
-
const DEFAULT_VIDEO_FRAMES = 33;
|
|
9
|
-
export function createDiffusionFitRequest(params) {
|
|
10
|
-
const config = (params.modelConfig ?? {});
|
|
11
|
-
const artifacts = params.artifacts ?? {};
|
|
12
|
-
if (config.mode === 'world') {
|
|
13
|
-
const taehv = artifacts['taehvModelPath'];
|
|
14
|
-
const scene = artifacts['seedScenePath'];
|
|
15
|
-
if (taehv === undefined || scene === undefined) {
|
|
16
|
-
return { supported: false, detail: 'ABot fit requires a resolved decoder and scene pack' };
|
|
17
|
-
}
|
|
18
|
-
const { fitSteps, session } = splitWorldConfig(config.world);
|
|
19
|
-
return {
|
|
20
|
-
supported: true,
|
|
21
|
-
probe: {
|
|
22
|
-
engine: 'diffusion-cpp',
|
|
23
|
-
request: {
|
|
24
|
-
mode: 'world',
|
|
25
|
-
files: { model: params.modelPath, taehv, scene },
|
|
26
|
-
config: session,
|
|
27
|
-
workload: { walkSteps: fitSteps }
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
if (config.mode === 'upscale') {
|
|
33
|
-
return {
|
|
34
|
-
supported: true,
|
|
35
|
-
probe: {
|
|
36
|
-
engine: 'diffusion-cpp',
|
|
37
|
-
request: {
|
|
38
|
-
mode: 'upscale',
|
|
39
|
-
files: { esrgan: params.modelPath },
|
|
40
|
-
config: toEsrganAddonConfig(config),
|
|
41
|
-
workload: { upscaleRepeats: 1 }
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
const upscaler = config.mode === 'video' ? undefined : config.upscaler;
|
|
47
|
-
const esrgan = upscaler === undefined ? undefined : artifacts['esrganModelPath'];
|
|
48
|
-
if (upscaler !== undefined && esrgan === undefined) {
|
|
49
|
-
return { supported: false, detail: 'the configured ESRGAN checkpoint is not resolved' };
|
|
50
|
-
}
|
|
51
|
-
const files = {
|
|
52
|
-
model: params.modelPath,
|
|
53
|
-
...(esrgan !== undefined && { esrgan }),
|
|
54
|
-
...(artifacts['clipLModelPath'] !== undefined && { clipL: artifacts['clipLModelPath'] }),
|
|
55
|
-
...(artifacts['clipGModelPath'] !== undefined && { clipG: artifacts['clipGModelPath'] }),
|
|
56
|
-
...(artifacts['t5XxlModelPath'] !== undefined && { t5Xxl: artifacts['t5XxlModelPath'] }),
|
|
57
|
-
...(artifacts['llmModelPath'] !== undefined && { llm: artifacts['llmModelPath'] }),
|
|
58
|
-
...(artifacts['projectionModelPath'] !== undefined && {
|
|
59
|
-
llmVision: artifacts['projectionModelPath']
|
|
60
|
-
}),
|
|
61
|
-
...(artifacts['vaeModelPath'] !== undefined && { vae: artifacts['vaeModelPath'] }),
|
|
62
|
-
...(artifacts['uncondModelPath'] !== undefined && {
|
|
63
|
-
uncondModel: artifacts['uncondModelPath']
|
|
64
|
-
}),
|
|
65
|
-
...(artifacts['clipVisionModelPath'] !== undefined && {
|
|
66
|
-
clipVision: artifacts['clipVisionModelPath']
|
|
67
|
-
}),
|
|
68
|
-
...(artifacts['audioVaeModelPath'] !== undefined && {
|
|
69
|
-
audioVae: artifacts['audioVaeModelPath']
|
|
70
|
-
}),
|
|
71
|
-
...(artifacts['embeddingsConnectorsModelPath'] !== undefined && {
|
|
72
|
-
embeddingsConnectors: artifacts['embeddingsConnectorsModelPath']
|
|
73
|
-
})
|
|
74
|
-
};
|
|
75
|
-
// `mode` and `upscaler` are this SDK's own keys; the engine config has no
|
|
76
|
-
// field for either.
|
|
77
|
-
const { mode: _mode, upscaler: _upscaler, ...engineConfig } = config;
|
|
78
|
-
return {
|
|
79
|
-
supported: true,
|
|
80
|
-
probe: {
|
|
81
|
-
engine: 'diffusion-cpp',
|
|
82
|
-
request: {
|
|
83
|
-
files,
|
|
84
|
-
config: { ...engineConfig, ...flattenUpscalerKeys(upscaler) },
|
|
85
|
-
workload: {
|
|
86
|
-
...(config.vae_tiling !== undefined && { vaeTiling: config.vae_tiling }),
|
|
87
|
-
...(config.mode === 'video' && { videoFrames: DEFAULT_VIDEO_FRAMES }),
|
|
88
|
-
...(upscaler !== undefined && { upscaleRepeats: 1 })
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
2
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
3
|
-
export type LlamaLoadKind = 'completion' | 'embedding';
|
|
4
|
-
export declare function llamaLoadKindFor(modelType: CanonicalModelType): LlamaLoadKind | undefined;
|
|
5
|
-
export interface LlamaFitRequestParams {
|
|
6
|
-
loadKind: LlamaLoadKind;
|
|
7
|
-
modelPath: string;
|
|
8
|
-
modelConfig: unknown;
|
|
9
|
-
artifacts?: Record<string, string> | undefined;
|
|
10
|
-
marginBytes?: number | undefined;
|
|
11
|
-
}
|
|
12
|
-
export declare function createLlamaFitRequest(params: LlamaFitRequestParams): FitRequestPlan;
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
-
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
-
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
|
-
export function llamaLoadKindFor(modelType) {
|
|
5
|
-
if (modelType === ModelType.llamacppCompletion)
|
|
6
|
-
return 'completion';
|
|
7
|
-
if (modelType === ModelType.llamacppEmbedding)
|
|
8
|
-
return 'embedding';
|
|
9
|
-
return undefined;
|
|
10
|
-
}
|
|
11
|
-
export function createLlamaFitRequest(params) {
|
|
12
|
-
const { loadKind } = params;
|
|
13
|
-
if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
|
|
14
|
-
return { supported: false, detail: 'multimodal projection loads are not representable' };
|
|
15
|
-
}
|
|
16
|
-
const modelConfig = (params.modelConfig ?? {});
|
|
17
|
-
// The assessment budgets against this machine's devices, so a load spread
|
|
18
|
-
// over remote GPUs is refused rather than measured against the wrong ones.
|
|
19
|
-
if (modelConfig['rpc-servers'] !== undefined) {
|
|
20
|
-
return { supported: false, detail: 'unsupported load setting: rpc-servers' };
|
|
21
|
-
}
|
|
22
|
-
const request = {
|
|
23
|
-
modelPath: params.modelPath,
|
|
24
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
25
|
-
};
|
|
26
|
-
if (loadKind === 'completion') {
|
|
27
|
-
// The addon's fitter rejects these as unknown arguments, and none changes
|
|
28
|
-
// what it projects: the image limit gates images at inference, and the
|
|
29
|
-
// prompt-cache keys size host-RAM and disk state the fitter does not model.
|
|
30
|
-
const { 'image-max-megapixels': _imageMaxMegapixels, cache_checkpoints: _cacheCheckpoints, cache_checkpoints_max_bytes: _cacheCheckpointsMaxBytes, cache_checkpoint_storage: _cacheCheckpointStorage, cache_checkpoint_dir: _cacheCheckpointDir, cache_ram_mib: _cacheRamMib, ...fitConfig } = modelConfig;
|
|
31
|
-
const config = transformLlmConfig(fitConfig);
|
|
32
|
-
// The fit reads the draft model (DFlash model or external MTP head) like the load does.
|
|
33
|
-
const draftModelPath = params.artifacts?.['draftModelPath'];
|
|
34
|
-
if (draftModelPath)
|
|
35
|
-
config['spec_draft_model'] = draftModelPath;
|
|
36
|
-
const llm = { ...request, config };
|
|
37
|
-
return { supported: true, probe: { engine: 'llm-llamacpp', request: llm } };
|
|
38
|
-
}
|
|
39
|
-
const embed = {
|
|
40
|
-
...request,
|
|
41
|
-
config: transformEmbedConfig(modelConfig)
|
|
42
|
-
};
|
|
43
|
-
return { supported: true, probe: { engine: 'embed-llamacpp', request: embed } };
|
|
44
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
|
-
export interface NmtFitRequestParams {
|
|
3
|
-
modelPath: string;
|
|
4
|
-
modelConfig: unknown;
|
|
5
|
-
artifacts?: Record<string, string> | undefined;
|
|
6
|
-
marginBytes?: number | undefined;
|
|
7
|
-
}
|
|
8
|
-
export declare function createNmtFitRequest(params: NmtFitRequestParams): FitRequestPlan;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { nmtConfigFor, nmtFilesFor } from '../../../plugins/builtin/nmtcpp-translation/load-files.js';
|
|
2
|
-
export function createNmtFitRequest(params) {
|
|
3
|
-
const config = (params.modelConfig ?? {});
|
|
4
|
-
const files = nmtFilesFor(params.modelPath, params.artifacts ?? {});
|
|
5
|
-
const request = {
|
|
6
|
-
files,
|
|
7
|
-
config: nmtConfigFor(config, files.pivotModel),
|
|
8
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
9
|
-
};
|
|
10
|
-
return { supported: true, probe: { engine: 'translation-nmtcpp', request } };
|
|
11
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
2
|
-
export interface OcrFitRequestParams {
|
|
3
|
-
/** The recognizer, which is the load's primary model. */
|
|
4
|
-
modelPath: string;
|
|
5
|
-
modelConfig: unknown;
|
|
6
|
-
artifacts?: Record<string, string> | undefined;
|
|
7
|
-
marginBytes?: number | undefined;
|
|
8
|
-
}
|
|
9
|
-
export declare function createOcrFitRequest(params: OcrFitRequestParams): FitRequestPlan;
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { ocrParamsFor } from '../../../plugins/builtin/ggml-ocr/params.js';
|
|
2
|
-
export function createOcrFitRequest(params) {
|
|
3
|
-
const detectorPath = params.artifacts?.['detectorModelPath'];
|
|
4
|
-
if (detectorPath === undefined) {
|
|
5
|
-
return { supported: false, detail: 'the ocr detector is not resolved' };
|
|
6
|
-
}
|
|
7
|
-
const config = (params.modelConfig ?? {});
|
|
8
|
-
const request = {
|
|
9
|
-
...ocrParamsFor(detectorPath, params.modelPath, config),
|
|
10
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
11
|
-
};
|
|
12
|
-
return { supported: true, probe: { engine: 'ocr-ggml', request } };
|
|
13
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { TtsFitWorkload } from '../../../schemas/assess-model-fit.js';
|
|
2
|
-
import type { FitRequestPlan } from '../../../model-fit/request/create.js';
|
|
3
|
-
export interface TtsFitRequestParams {
|
|
4
|
-
ttsWorkload?: TtsFitWorkload | undefined;
|
|
5
|
-
modelPath: string;
|
|
6
|
-
modelConfig: unknown;
|
|
7
|
-
artifacts?: Record<string, string> | undefined;
|
|
8
|
-
marginBytes?: number | undefined;
|
|
9
|
-
}
|
|
10
|
-
export declare function createTtsFitRequest(params: TtsFitRequestParams): FitRequestPlan;
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
import fs from 'bare-fs';
|
|
2
|
-
import path from 'bare-path';
|
|
3
|
-
import { gpuLayersFromCount } from '../../../model-fit/request/gpu.js';
|
|
4
|
-
function unsupported(detail) {
|
|
5
|
-
return { supported: false, detail };
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* CosyVoice's companion set co-locates the flow, HiFT and voice GGUFs beside
|
|
9
|
-
* the LLM checkpoint, and the loader finds them by name in that directory. The
|
|
10
|
-
* fitter takes explicit paths, so the same directory is read here.
|
|
11
|
-
*/
|
|
12
|
-
function cosyvoiceCompanion(entries, dir, matches) {
|
|
13
|
-
const found = entries.find(matches);
|
|
14
|
-
return found === undefined ? undefined : path.join(dir, found);
|
|
15
|
-
}
|
|
16
|
-
function cosyvoiceRequest(modelPath, common) {
|
|
17
|
-
const dir = path.dirname(modelPath);
|
|
18
|
-
let entries;
|
|
19
|
-
try {
|
|
20
|
-
entries = fs.readdirSync(dir);
|
|
21
|
-
}
|
|
22
|
-
catch {
|
|
23
|
-
return unsupported('the cosyvoice companion set is not laid out beside the checkpoint');
|
|
24
|
-
}
|
|
25
|
-
const flowPath = cosyvoiceCompanion(entries, dir, (name) => name.includes('-flow-'));
|
|
26
|
-
const hiftPath = cosyvoiceCompanion(entries, dir, (name) => name.includes('-hift-'));
|
|
27
|
-
const voicePath = cosyvoiceCompanion(entries, dir, (name) => name === 'voice.gguf');
|
|
28
|
-
if (flowPath === undefined || hiftPath === undefined || voicePath === undefined) {
|
|
29
|
-
return unsupported('the cosyvoice companion set is not laid out beside the checkpoint');
|
|
30
|
-
}
|
|
31
|
-
return {
|
|
32
|
-
supported: true,
|
|
33
|
-
probe: {
|
|
34
|
-
engine: 'tts-ggml',
|
|
35
|
-
request: {
|
|
36
|
-
engineType: 'cosyvoice3',
|
|
37
|
-
cosyvoiceLlmModelPath: modelPath,
|
|
38
|
-
cosyvoiceFlowModelPath: flowPath,
|
|
39
|
-
cosyvoiceHiftModelPath: hiftPath,
|
|
40
|
-
cosyvoiceVoiceModelPath: voicePath,
|
|
41
|
-
...common
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
export function createTtsFitRequest(params) {
|
|
47
|
-
const config = (params.modelConfig ?? {});
|
|
48
|
-
const artifacts = params.artifacts ?? {};
|
|
49
|
-
const lavasrEnhancerPath = artifacts['lavasrEnhancerPath'];
|
|
50
|
-
const lavasrDenoiserPath = artifacts['lavasrDenoiserPath'];
|
|
51
|
-
const common = {
|
|
52
|
-
nGpuLayers: gpuLayersFromCount(config.useGPU, config.nGpuLayers),
|
|
53
|
-
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
|
|
54
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes }),
|
|
55
|
-
...(lavasrEnhancerPath !== undefined && { lavasrEnhancerPath }),
|
|
56
|
-
...(lavasrDenoiserPath !== undefined && { lavasrDenoiserPath })
|
|
57
|
-
};
|
|
58
|
-
const supported = (request) => ({
|
|
59
|
-
supported: true,
|
|
60
|
-
probe: { engine: 'tts-ggml', request }
|
|
61
|
-
});
|
|
62
|
-
switch (config.ttsEngine) {
|
|
63
|
-
case 'supertonic':
|
|
64
|
-
return supported({
|
|
65
|
-
engineType: 'supertonic',
|
|
66
|
-
supertonicModelPath: params.modelPath,
|
|
67
|
-
...common,
|
|
68
|
-
...(config.ttsNumInferenceSteps !== undefined && { steps: config.ttsNumInferenceSteps })
|
|
69
|
-
});
|
|
70
|
-
case 'parler':
|
|
71
|
-
return supported({
|
|
72
|
-
engineType: 'parler',
|
|
73
|
-
parlerModelPath: params.modelPath,
|
|
74
|
-
...common,
|
|
75
|
-
...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
|
|
76
|
-
});
|
|
77
|
-
case 'audio8': {
|
|
78
|
-
const codecDecoderPath = artifacts['audio8CodecDecoderPath'];
|
|
79
|
-
if (codecDecoderPath === undefined) {
|
|
80
|
-
return unsupported('the audio8 codec decoder is not resolved');
|
|
81
|
-
}
|
|
82
|
-
const codecEncoderPath = artifacts['audio8CodecEncoderPath'];
|
|
83
|
-
return supported({
|
|
84
|
-
engineType: 'audio8',
|
|
85
|
-
audio8LmPath: params.modelPath,
|
|
86
|
-
audio8CodecDecoderPath: codecDecoderPath,
|
|
87
|
-
...(codecEncoderPath !== undefined && { audio8CodecEncoderPath: codecEncoderPath }),
|
|
88
|
-
...common,
|
|
89
|
-
...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
case 'cosyvoice3':
|
|
93
|
-
return cosyvoiceRequest(params.modelPath, common);
|
|
94
|
-
case 'moss': {
|
|
95
|
-
const decoder = artifacts['mossCodecDecoderPath'];
|
|
96
|
-
const encoder = artifacts['mossCodecEncoderPath'];
|
|
97
|
-
if (!decoder)
|
|
98
|
-
return unsupported('the moss codec decoder is not resolved');
|
|
99
|
-
if (!params.ttsWorkload || !('promptRows' in params.ttsWorkload)) {
|
|
100
|
-
return unsupported('MOSS fit requires ttsWorkload with promptRows, referenceSamples and streaming');
|
|
101
|
-
}
|
|
102
|
-
if (params.ttsWorkload.referenceSamples > 0 && !encoder) {
|
|
103
|
-
return unsupported('MOSS reference encoding requires the codec encoder');
|
|
104
|
-
}
|
|
105
|
-
return supported({
|
|
106
|
-
engineType: 'moss',
|
|
107
|
-
mossBackbonePath: params.modelPath,
|
|
108
|
-
mossCodecDecoderPath: decoder,
|
|
109
|
-
...(encoder ? { mossCodecEncoderPath: encoder } : {}),
|
|
110
|
-
...common,
|
|
111
|
-
...params.ttsWorkload,
|
|
112
|
-
...(config.threads !== undefined ? { threads: config.threads } : {}),
|
|
113
|
-
...(config.streamChunkTokens !== undefined
|
|
114
|
-
? { streamChunkTokens: config.streamChunkTokens }
|
|
115
|
-
: {}),
|
|
116
|
-
...(config.durationTokens !== undefined ? { durationTokens: config.durationTokens } : {})
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
case 'moss-sfx': {
|
|
120
|
-
if (!params.ttsWorkload || !('prompt' in params.ttsWorkload)) {
|
|
121
|
-
return unsupported('MOSS-SoundEffect fit requires ttsWorkload with prompt and seconds');
|
|
122
|
-
}
|
|
123
|
-
return supported({
|
|
124
|
-
engineType: 'moss-sfx',
|
|
125
|
-
mossSoundEffectPath: params.modelPath,
|
|
126
|
-
...common,
|
|
127
|
-
...params.ttsWorkload,
|
|
128
|
-
...(config.negativePrompt !== undefined ? { negativePrompt: config.negativePrompt } : {}),
|
|
129
|
-
...(config.steps !== undefined ? { steps: config.steps } : {}),
|
|
130
|
-
...(config.guidance !== undefined ? { guidance: config.guidance } : {}),
|
|
131
|
-
...(config.shift !== undefined ? { shift: config.shift } : {}),
|
|
132
|
-
...(config.threads !== undefined ? { threads: config.threads } : {})
|
|
133
|
-
});
|
|
134
|
-
}
|
|
135
|
-
case 'moss-speech':
|
|
136
|
-
return unsupported('the addon does not provide MOSS-Speech memory fit');
|
|
137
|
-
default: {
|
|
138
|
-
const s3genPath = artifacts['s3genPath'];
|
|
139
|
-
if (s3genPath === undefined) {
|
|
140
|
-
return unsupported('the chatterbox s3gen checkpoint is not resolved');
|
|
141
|
-
}
|
|
142
|
-
return supported({
|
|
143
|
-
engineType: 'chatterbox',
|
|
144
|
-
t3ModelPath: params.modelPath,
|
|
145
|
-
s3genModelPath: s3genPath,
|
|
146
|
-
...common,
|
|
147
|
-
...(config.nCtx !== undefined && config.nCtx > 0 && { nCtx: config.nCtx }),
|
|
148
|
-
...(config.kvCacheType !== undefined && { kvCacheType: config.kvCacheType })
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
}
|