@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
/** The llama fitters append a host row after the devices they assigned to. */
|
|
2
|
-
const HOST_ROW = 'host';
|
|
3
|
-
/** ggml's own name for the CPU backend device, which the fitter may assign to. */
|
|
4
|
-
const CPU_DEVICE = 'CPU';
|
|
5
|
-
function isLlama(probe) {
|
|
6
|
-
return probe.engine === 'llm-llamacpp' || probe.engine === 'embed-llamacpp';
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* `error` is the engine reporting it could not read the model or find a backend
|
|
10
|
-
* device: absence of evidence, not evidence of insufficiency.
|
|
11
|
-
*/
|
|
12
|
-
function verdictFor(status) {
|
|
13
|
-
if (status === 'fits')
|
|
14
|
-
return 'fit';
|
|
15
|
-
if (status === 'does-not-fit')
|
|
16
|
-
return 'does-not-fit';
|
|
17
|
-
return 'unknown';
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* The diffusion fitter reports a placement rather than a reason string.
|
|
21
|
-
* `changed` means it reached a fitting arrangement only by moving modules
|
|
22
|
-
* between backends, which is why the status is already `does-not-fit`.
|
|
23
|
-
*/
|
|
24
|
-
function diffusionReason(result) {
|
|
25
|
-
if (result.status === 'fits')
|
|
26
|
-
return 'fits';
|
|
27
|
-
if (result.status === 'error')
|
|
28
|
-
return 'model-unreadable';
|
|
29
|
-
return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
|
|
30
|
-
}
|
|
31
|
-
function reasonFor(probe) {
|
|
32
|
-
return probe.engine === 'diffusion-cpp' ? diffusionReason(probe.result) : probe.result.reason;
|
|
33
|
-
}
|
|
34
|
-
function sumOf(...values) {
|
|
35
|
-
const present = values.filter((value) => value !== undefined);
|
|
36
|
-
return present.length === 0 ? undefined : present.reduce((total, value) => total + value, 0);
|
|
37
|
-
}
|
|
38
|
-
function breakdown(weights, context, compute) {
|
|
39
|
-
return {
|
|
40
|
-
...(weights !== undefined && { weightsBytes: weights }),
|
|
41
|
-
...(context !== undefined && { contextBytes: context }),
|
|
42
|
-
...(compute !== undefined && { computeBytes: compute })
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* The parts of `deviceBytes` the engine separates. Audiogen reports a peak
|
|
47
|
-
* across pipeline phases and diffusion a per-module table, neither of which
|
|
48
|
-
* divides into these three, so both come back empty. OCR sizes its weights
|
|
49
|
-
* alone and leaves the image-dependent memory uncounted.
|
|
50
|
-
*
|
|
51
|
-
* The llama rows are summed over the same devices the engine's own
|
|
52
|
-
* `deviceBytes` covers, which is every row but `host`.
|
|
53
|
-
*/
|
|
54
|
-
function breakdownFor(probe) {
|
|
55
|
-
if (isLlama(probe)) {
|
|
56
|
-
const devices = probe.result.devices.filter((device) => device.name !== HOST_ROW);
|
|
57
|
-
return breakdown(sumOf(...devices.map((device) => device.modelBytes)), sumOf(...devices.map((device) => device.contextBytes)), sumOf(...devices.map((device) => device.computeBytes)));
|
|
58
|
-
}
|
|
59
|
-
// Whisper names its parts `kvBytes` and `computeBytes`, parakeet splits
|
|
60
|
-
// compute across encoder and decoder; an engine fills one set or the other.
|
|
61
|
-
if (probe.engine === 'asr-ggml') {
|
|
62
|
-
const result = probe.result;
|
|
63
|
-
// A VAD model's weights are inside `deviceBytes`, so they belong in the
|
|
64
|
-
// weights the breakdown divides it into.
|
|
65
|
-
return breakdown(sumOf(result.weightsBytes, result.vadBytes), sumOf(result.kvBytes, result.decoderStateBytes), sumOf(result.computeBytes, result.encoderComputeBytes, result.decoderComputeBytes));
|
|
66
|
-
}
|
|
67
|
-
if (probe.engine === 'bci-whispercpp') {
|
|
68
|
-
const result = probe.result;
|
|
69
|
-
return breakdown(result.weightsBytes, result.kvBytes, result.computeBytes);
|
|
70
|
-
}
|
|
71
|
-
if (probe.engine === 'tts-ggml') {
|
|
72
|
-
const result = probe.result;
|
|
73
|
-
return breakdown(result.weightsBytes, result.stateBytes, sumOf(result.lmComputeBytes, result.codecComputeBytes));
|
|
74
|
-
}
|
|
75
|
-
if (probe.engine === 'ocr-ggml') {
|
|
76
|
-
return breakdown(probe.result.weightsBytes, undefined, undefined);
|
|
77
|
-
}
|
|
78
|
-
return {};
|
|
79
|
-
}
|
|
80
|
-
function llamaProjection(probe) {
|
|
81
|
-
const first = probe.result.devices.find((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE);
|
|
82
|
-
const devices = probe.result.devices
|
|
83
|
-
.filter((device) => device.name !== HOST_ROW)
|
|
84
|
-
.map((device) => ({
|
|
85
|
-
name: device.name,
|
|
86
|
-
totalBytes: device.totalBytes,
|
|
87
|
-
freeBytes: device.freeBytes,
|
|
88
|
-
weightsBytes: device.modelBytes,
|
|
89
|
-
contextBytes: device.contextBytes,
|
|
90
|
-
computeBytes: device.computeBytes
|
|
91
|
-
}));
|
|
92
|
-
return {
|
|
93
|
-
deviceBytes: probe.result.deviceBytes,
|
|
94
|
-
hostBytes: probe.result.hostBytes,
|
|
95
|
-
...breakdownFor(probe),
|
|
96
|
-
...(devices.length > 0 && { devices }),
|
|
97
|
-
...(first !== undefined && {
|
|
98
|
-
deviceName: first.name,
|
|
99
|
-
deviceFreeBytes: first.freeBytes,
|
|
100
|
-
deviceTotalBytes: first.totalBytes
|
|
101
|
-
})
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* The translation fitter reports one allowance against the free memory of the
|
|
106
|
-
* device it ran on. Only the file bytes inside it are weights; the rest does
|
|
107
|
-
* not divide into context and compute.
|
|
108
|
-
*/
|
|
109
|
-
function nmtProjection(result) {
|
|
110
|
-
return {
|
|
111
|
-
deviceName: result.backend,
|
|
112
|
-
deviceBytes: result.requiredBytes,
|
|
113
|
-
deviceFreeBytes: result.freeBytes,
|
|
114
|
-
weightsBytes: result.modelBytes,
|
|
115
|
-
report: result.report
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
function projectionFor(probe) {
|
|
119
|
-
if (probe.engine === 'diffusion-cpp') {
|
|
120
|
-
return { deviceName: probe.result.backend, report: probe.result.report };
|
|
121
|
-
}
|
|
122
|
-
if (probe.engine === 'translation-nmtcpp')
|
|
123
|
-
return nmtProjection(probe.result);
|
|
124
|
-
if (isLlama(probe))
|
|
125
|
-
return llamaProjection(probe);
|
|
126
|
-
const result = probe.result;
|
|
127
|
-
return {
|
|
128
|
-
deviceName: result.deviceName,
|
|
129
|
-
deviceBytes: result.deviceBytes,
|
|
130
|
-
hostBytes: result.hostBytes,
|
|
131
|
-
deviceFreeBytes: result.deviceFreeBytes,
|
|
132
|
-
deviceTotalBytes: result.deviceTotalBytes,
|
|
133
|
-
report: result.report,
|
|
134
|
-
...breakdownFor(probe)
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
/**
|
|
138
|
-
* Only the llama fitters resolve a placement; the other engines' loads are not
|
|
139
|
-
* layer-sliced across devices.
|
|
140
|
-
*
|
|
141
|
-
* `nGpuDevices` counts the devices the weights were actually placed on, less
|
|
142
|
-
* the CPU backend device, which the fitter assigns to like any other but whose
|
|
143
|
-
* demand is host memory.
|
|
144
|
-
*/
|
|
145
|
-
function planFor(probe) {
|
|
146
|
-
if (!isLlama(probe))
|
|
147
|
-
return undefined;
|
|
148
|
-
const assigned = probe.result.devices.filter((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE && device.modelBytes > 0);
|
|
149
|
-
return {
|
|
150
|
-
nCtx: probe.result.ctxSize,
|
|
151
|
-
nGpuLayers: probe.result.gpuLayers,
|
|
152
|
-
nGpuDevices: assigned.length
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
export function classifyFit(probe, provenance) {
|
|
156
|
-
const verdict = verdictFor(probe.result.status);
|
|
157
|
-
const plan = verdict === 'fit' ? planFor(probe) : undefined;
|
|
158
|
-
return {
|
|
159
|
-
...provenance,
|
|
160
|
-
engine: probe.engine,
|
|
161
|
-
verdict,
|
|
162
|
-
reason: reasonFor(probe),
|
|
163
|
-
...(plan !== undefined && { plan }),
|
|
164
|
-
...(verdict !== 'unknown' && { projection: projectionFor(probe) })
|
|
165
|
-
};
|
|
166
|
-
}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { TtsFitWorkload } from '../../../schemas/assess-model-fit.js';
|
|
2
|
-
import { type CanonicalModelType, type TranscriptionFitWorkload } from '../../../schemas/index.js';
|
|
3
|
-
import type { FitProbeRequest } from './engine-fit.js';
|
|
4
|
-
export type FitRequestPlan = {
|
|
5
|
-
supported: true;
|
|
6
|
-
probe: FitProbeRequest;
|
|
7
|
-
} | {
|
|
8
|
-
supported: false;
|
|
9
|
-
detail: string;
|
|
10
|
-
};
|
|
11
|
-
export interface CreateFitRequestParams {
|
|
12
|
-
ttsWorkload?: TtsFitWorkload | undefined;
|
|
13
|
-
modelType: CanonicalModelType;
|
|
14
|
-
modelPath: string;
|
|
15
|
-
modelConfig: unknown;
|
|
16
|
-
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
17
|
-
artifacts?: Record<string, string> | undefined;
|
|
18
|
-
isShardedModel: boolean;
|
|
19
|
-
/** Free memory that must remain for the projection to count as fitting. */
|
|
20
|
-
marginBytes?: number | undefined;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Builds a fit request from the same resolved config and artifacts the real
|
|
24
|
-
* load is about to use, or explains why this load cannot be answered for.
|
|
25
|
-
*
|
|
26
|
-
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
27
|
-
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
28
|
-
* — are refused here so no engine is asked a question it cannot answer.
|
|
29
|
-
* Value-level policy stays inside each engine's own fitter.
|
|
30
|
-
*/
|
|
31
|
-
export declare function createFitRequest(params: CreateFitRequestParams): FitRequestPlan;
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
-
import { createBciFitRequest, createMossTranscribeFitRequest, createParakeetFitRequest, createWhisperFitRequest } from './engines/asr.js';
|
|
3
|
-
import { createAudiogenFitRequest } from './engines/audiogen.js';
|
|
4
|
-
import { createDiffusionFitRequest } from './engines/diffusion.js';
|
|
5
|
-
import { createLlamaFitRequest, llamaLoadKindFor } from './engines/llama.js';
|
|
6
|
-
import { createNmtFitRequest } from './engines/nmt.js';
|
|
7
|
-
import { createOcrFitRequest } from './engines/ocr.js';
|
|
8
|
-
import { createTtsFitRequest } from './engines/tts.js';
|
|
9
|
-
/**
|
|
10
|
-
* Builds a fit request from the same resolved config and artifacts the real
|
|
11
|
-
* load is about to use, or explains why this load cannot be answered for.
|
|
12
|
-
*
|
|
13
|
-
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
14
|
-
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
15
|
-
* — are refused here so no engine is asked a question it cannot answer.
|
|
16
|
-
* Value-level policy stays inside each engine's own fitter.
|
|
17
|
-
*/
|
|
18
|
-
export function createFitRequest(params) {
|
|
19
|
-
const { modelType, modelPath, modelConfig, artifacts, marginBytes } = params;
|
|
20
|
-
if (params.transcriptionWorkload !== undefined && modelType !== ModelType.mossTranscribe) {
|
|
21
|
-
return {
|
|
22
|
-
supported: false,
|
|
23
|
-
detail: 'transcriptionWorkload is supported only for MOSS transcription'
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
// Every engine reads whole files rather than a shard set, so a split model is
|
|
27
|
-
// not a shape any of them can be pointed at.
|
|
28
|
-
if (params.isShardedModel) {
|
|
29
|
-
return { supported: false, detail: 'sharded models are not representable' };
|
|
30
|
-
}
|
|
31
|
-
const loadKind = llamaLoadKindFor(modelType);
|
|
32
|
-
if (loadKind !== undefined) {
|
|
33
|
-
return createLlamaFitRequest({ loadKind, modelPath, modelConfig, artifacts, marginBytes });
|
|
34
|
-
}
|
|
35
|
-
const common = { modelPath, modelConfig, artifacts, marginBytes };
|
|
36
|
-
switch (modelType) {
|
|
37
|
-
case ModelType.mossTranscribe:
|
|
38
|
-
return createMossTranscribeFitRequest({
|
|
39
|
-
...common,
|
|
40
|
-
transcriptionWorkload: params.transcriptionWorkload
|
|
41
|
-
});
|
|
42
|
-
case ModelType.whispercppTranscription:
|
|
43
|
-
return createWhisperFitRequest(common);
|
|
44
|
-
case ModelType.parakeetTranscription:
|
|
45
|
-
return createParakeetFitRequest(common);
|
|
46
|
-
case ModelType.bciWhispercppTranscription:
|
|
47
|
-
return createBciFitRequest(common);
|
|
48
|
-
case ModelType.ttsGgml:
|
|
49
|
-
return createTtsFitRequest({ ...common, ttsWorkload: params.ttsWorkload });
|
|
50
|
-
case ModelType.audiogenGgml:
|
|
51
|
-
return createAudiogenFitRequest(common);
|
|
52
|
-
case ModelType.sdcppGeneration:
|
|
53
|
-
return createDiffusionFitRequest({ modelPath, modelConfig, artifacts });
|
|
54
|
-
case ModelType.ggmlOcr:
|
|
55
|
-
return createOcrFitRequest(common);
|
|
56
|
-
case ModelType.nmtcppTranslation:
|
|
57
|
-
return createNmtFitRequest(common);
|
|
58
|
-
default:
|
|
59
|
-
return { supported: false, detail: `no fitter for model type: ${modelType}` };
|
|
60
|
-
}
|
|
61
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
import type { AsrFitRequest, AsrFitResult } from '@qvac/asr-ggml';
|
|
2
|
-
import type { AudiogenFitRequest, AudiogenFitResult } from '@qvac/audiogen-ggml';
|
|
3
|
-
import type { BciFitRequest, BciFitResult } from '@qvac/bci-whispercpp';
|
|
4
|
-
import type { DiffusionFitRequest, DiffusionFitResult } from '@qvac/diffusion-cpp';
|
|
5
|
-
import type { EmbedFitRequest, EmbedFitResult } from '@qvac/embed-llamacpp';
|
|
6
|
-
import type LlmLlamacpp from '@qvac/llm-llamacpp';
|
|
7
|
-
import type { OcrFitRequest, OcrFitResult } from '@qvac/ocr-ggml';
|
|
8
|
-
import type TranslationNmtcpp from '@qvac/translation-nmtcpp';
|
|
9
|
-
import type { TtsFitRequest, TtsFitResult } from '@qvac/tts-ggml';
|
|
10
|
-
/**
|
|
11
|
-
* The engine packages each expose their own `assessFit`, taking the same fields
|
|
12
|
-
* their loader takes and returning the figures their fitter measures. Every
|
|
13
|
-
* member is plain JSON, so a request can cross a process boundary unchanged.
|
|
14
|
-
*/
|
|
15
|
-
export type FitProbeRequest = {
|
|
16
|
-
engine: 'llm-llamacpp';
|
|
17
|
-
request: LlmLlamacpp.FitRequest;
|
|
18
|
-
} | {
|
|
19
|
-
engine: 'embed-llamacpp';
|
|
20
|
-
request: EmbedFitRequest;
|
|
21
|
-
} | {
|
|
22
|
-
engine: 'asr-ggml';
|
|
23
|
-
request: AsrFitRequest;
|
|
24
|
-
} | {
|
|
25
|
-
engine: 'bci-whispercpp';
|
|
26
|
-
request: BciFitRequest;
|
|
27
|
-
} | {
|
|
28
|
-
engine: 'tts-ggml';
|
|
29
|
-
request: TtsFitRequest;
|
|
30
|
-
} | {
|
|
31
|
-
engine: 'audiogen-ggml';
|
|
32
|
-
request: AudiogenFitRequest;
|
|
33
|
-
} | {
|
|
34
|
-
engine: 'diffusion-cpp';
|
|
35
|
-
request: DiffusionFitRequest;
|
|
36
|
-
} | {
|
|
37
|
-
engine: 'ocr-ggml';
|
|
38
|
-
request: OcrFitRequest;
|
|
39
|
-
} | {
|
|
40
|
-
engine: 'translation-nmtcpp';
|
|
41
|
-
request: TranslationNmtcpp.FitRequest;
|
|
42
|
-
};
|
|
43
|
-
export type FitProbeEngine = FitProbeRequest['engine'];
|
|
44
|
-
export type FitProbeResult = {
|
|
45
|
-
engine: 'llm-llamacpp';
|
|
46
|
-
result: LlmLlamacpp.FitResult;
|
|
47
|
-
} | {
|
|
48
|
-
engine: 'embed-llamacpp';
|
|
49
|
-
result: EmbedFitResult;
|
|
50
|
-
} | {
|
|
51
|
-
engine: 'asr-ggml';
|
|
52
|
-
result: AsrFitResult;
|
|
53
|
-
} | {
|
|
54
|
-
engine: 'bci-whispercpp';
|
|
55
|
-
result: BciFitResult;
|
|
56
|
-
} | {
|
|
57
|
-
engine: 'tts-ggml';
|
|
58
|
-
result: TtsFitResult;
|
|
59
|
-
} | {
|
|
60
|
-
engine: 'audiogen-ggml';
|
|
61
|
-
result: AudiogenFitResult;
|
|
62
|
-
} | {
|
|
63
|
-
engine: 'diffusion-cpp';
|
|
64
|
-
result: DiffusionFitResult;
|
|
65
|
-
} | {
|
|
66
|
-
engine: 'ocr-ggml';
|
|
67
|
-
result: OcrFitResult;
|
|
68
|
-
} | {
|
|
69
|
-
engine: 'translation-nmtcpp';
|
|
70
|
-
result: TranslationNmtcpp.FitResult;
|
|
71
|
-
};
|
|
72
|
-
/** Calls one engine's fitter, taken from the plugin that owns its addon. */
|
|
73
|
-
export declare function callEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
/** Calls one engine's fitter, taken from the plugin that owns its addon. */
|
|
2
|
-
export async function callEngineFit(probe) {
|
|
3
|
-
const { getAllPlugins } = await import('../../../plugins/registry.js');
|
|
4
|
-
const addonPackage = `@qvac/${probe.engine}`;
|
|
5
|
-
const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === addonPackage);
|
|
6
|
-
if (plugin?.assessFit === undefined) {
|
|
7
|
-
throw new TypeError(`no registered plugin exposes assessFit for ${addonPackage}`);
|
|
8
|
-
}
|
|
9
|
-
return {
|
|
10
|
-
engine: probe.engine,
|
|
11
|
-
result: await plugin.assessFit(probe.request)
|
|
12
|
-
};
|
|
13
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { TranscriptionFitWorkload } from '../../../../schemas/index.js';
|
|
2
|
-
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
3
|
-
export interface AsrFitRequestParams {
|
|
4
|
-
modelPath: string;
|
|
5
|
-
modelConfig: unknown;
|
|
6
|
-
artifacts?: Record<string, string> | undefined;
|
|
7
|
-
marginBytes?: number | undefined;
|
|
8
|
-
}
|
|
9
|
-
export declare function createMossTranscribeFitRequest(params: AsrFitRequestParams & {
|
|
10
|
-
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
11
|
-
}): FitRequestPlan;
|
|
12
|
-
export declare function createWhisperFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
13
|
-
export declare function createParakeetFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
14
|
-
export declare function createBciFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
import { gpuLayersFromGate } from './gpu.js';
|
|
2
|
-
export function createMossTranscribeFitRequest(params) {
|
|
3
|
-
if (params.transcriptionWorkload === undefined) {
|
|
4
|
-
return {
|
|
5
|
-
supported: false,
|
|
6
|
-
detail: 'MOSS fit requires transcriptionWorkload with explicit audioSeconds'
|
|
7
|
-
};
|
|
8
|
-
}
|
|
9
|
-
const config = (params.modelConfig ?? {});
|
|
10
|
-
const workload = params.transcriptionWorkload;
|
|
11
|
-
const request = {
|
|
12
|
-
engine: 'moss-transcribe',
|
|
13
|
-
modelPath: params.modelPath,
|
|
14
|
-
audioSeconds: workload.audioSeconds,
|
|
15
|
-
...(workload.prompt !== undefined && { prompt: workload.prompt }),
|
|
16
|
-
...(workload.hotwords !== undefined && { hotwords: workload.hotwords }),
|
|
17
|
-
...(workload.maxNewTokens !== undefined && { maxNewTokens: workload.maxNewTokens }),
|
|
18
|
-
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
19
|
-
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
20
|
-
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
|
|
21
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
22
|
-
};
|
|
23
|
-
return { supported: true, probe: { engine: 'asr-ggml', request } };
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* The longest single transcribe the projection has to cover. Whisper's KV cache
|
|
27
|
-
* and decode graph are sized from it; unset leaves the engine's own default,
|
|
28
|
-
* which is the worst case it supports.
|
|
29
|
-
*/
|
|
30
|
-
function audioSeconds(durationMs) {
|
|
31
|
-
if (durationMs === undefined || durationMs <= 0)
|
|
32
|
-
return undefined;
|
|
33
|
-
return durationMs / 1000;
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Worst-case resident decoders. Each strategy carries its own count and `-1` is
|
|
37
|
-
* the engine's "use the default" sentinel rather than a number of decoders.
|
|
38
|
-
*/
|
|
39
|
-
function decoderCount(candidates) {
|
|
40
|
-
for (const candidate of candidates) {
|
|
41
|
-
if (candidate !== undefined && candidate > 0)
|
|
42
|
-
return candidate;
|
|
43
|
-
}
|
|
44
|
-
return undefined;
|
|
45
|
-
}
|
|
46
|
-
export function createWhisperFitRequest(params) {
|
|
47
|
-
const config = (params.modelConfig ?? {});
|
|
48
|
-
const vadModelPath = params.artifacts?.['vadModelPath'];
|
|
49
|
-
const decoders = decoderCount([
|
|
50
|
-
config.strategy === 'beam_search' ? config.beam_search_beam_size : config.greedy_best_of
|
|
51
|
-
]);
|
|
52
|
-
const seconds = audioSeconds(config.duration_ms);
|
|
53
|
-
const request = {
|
|
54
|
-
engine: 'whisper',
|
|
55
|
-
modelPath: params.modelPath,
|
|
56
|
-
gpuLayers: gpuLayersFromGate(config.contextParams?.use_gpu),
|
|
57
|
-
...(vadModelPath !== undefined && { vadModelPath }),
|
|
58
|
-
...(config.contextParams?.flash_attn !== undefined && {
|
|
59
|
-
flashAttn: config.contextParams.flash_attn
|
|
60
|
-
}),
|
|
61
|
-
...(config.contextParams?.gpu_device !== undefined && {
|
|
62
|
-
gpuDevice: config.contextParams.gpu_device
|
|
63
|
-
}),
|
|
64
|
-
...(decoders !== undefined && { decoders }),
|
|
65
|
-
...(seconds !== undefined && { audioSeconds: seconds }),
|
|
66
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
67
|
-
};
|
|
68
|
-
return { supported: true, probe: { engine: 'asr-ggml', request } };
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* The encoder's left context and Sortformer's rolling history size its live
|
|
72
|
-
* state, and the fit request measures that state in frames while the load sets
|
|
73
|
-
* it in milliseconds. Converting needs the model's frame rate, which is a
|
|
74
|
-
* property of the checkpoint, so a load that pins either is refused rather than
|
|
75
|
-
* projected against the engine's default window.
|
|
76
|
-
*/
|
|
77
|
-
const UNREPRESENTABLE_STREAMING = [
|
|
78
|
-
'streamingLeftContextMs',
|
|
79
|
-
'streamingHistoryMs'
|
|
80
|
-
];
|
|
81
|
-
export function createParakeetFitRequest(params) {
|
|
82
|
-
const config = (params.modelConfig ?? {});
|
|
83
|
-
if (config.streaming === true) {
|
|
84
|
-
for (const key of UNREPRESENTABLE_STREAMING) {
|
|
85
|
-
if (config[key] !== undefined) {
|
|
86
|
-
return { supported: false, detail: `load setting the projection cannot represent: ${key}` };
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
const request = {
|
|
91
|
-
engine: 'parakeet',
|
|
92
|
-
modelPath: params.modelPath,
|
|
93
|
-
gpuLayers: gpuLayersFromGate(config.useGPU),
|
|
94
|
-
...(config.maxThreads !== undefined && { threads: config.maxThreads }),
|
|
95
|
-
// Only a streaming load holds a live session, so outside one the chunk
|
|
96
|
-
// cadence describes no allocation the projection should cover.
|
|
97
|
-
...(config.streaming === true &&
|
|
98
|
-
config.streamingChunkMs !== undefined && { nemotronChunkMs: config.streamingChunkMs }),
|
|
99
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
100
|
-
};
|
|
101
|
-
return { supported: true, probe: { engine: 'asr-ggml', request } };
|
|
102
|
-
}
|
|
103
|
-
export function createBciFitRequest(params) {
|
|
104
|
-
const config = (params.modelConfig ?? {});
|
|
105
|
-
const embedderPath = params.artifacts?.['embedderPath'];
|
|
106
|
-
const whisper = config.whisperConfig;
|
|
107
|
-
const decoders = decoderCount([whisper?.beam_search_beam_size, whisper?.greedy_best_of]);
|
|
108
|
-
const seconds = audioSeconds(whisper?.duration_ms);
|
|
109
|
-
return {
|
|
110
|
-
supported: true,
|
|
111
|
-
probe: {
|
|
112
|
-
engine: 'bci-whispercpp',
|
|
113
|
-
request: {
|
|
114
|
-
modelPath: params.modelPath,
|
|
115
|
-
...(embedderPath !== undefined && { embedderPath }),
|
|
116
|
-
gpuLayers: gpuLayersFromGate(config.contextParams?.use_gpu ?? true),
|
|
117
|
-
...(config.contextParams?.gpu_device !== undefined && {
|
|
118
|
-
gpuDevice: config.contextParams.gpu_device
|
|
119
|
-
}),
|
|
120
|
-
...(decoders !== undefined && { decoders }),
|
|
121
|
-
...(seconds !== undefined && { audioSeconds: seconds }),
|
|
122
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
};
|
|
126
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
2
|
-
export interface AudiogenFitRequestParams {
|
|
3
|
-
modelPath: string;
|
|
4
|
-
modelConfig: unknown;
|
|
5
|
-
artifacts?: Record<string, string> | undefined;
|
|
6
|
-
marginBytes?: number | undefined;
|
|
7
|
-
}
|
|
8
|
-
export declare function createAudiogenFitRequest(params: AudiogenFitRequestParams): FitRequestPlan;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
2
|
-
export interface DiffusionFitRequestParams {
|
|
3
|
-
modelPath: string;
|
|
4
|
-
modelConfig: unknown;
|
|
5
|
-
artifacts?: Record<string, string> | undefined;
|
|
6
|
-
}
|
|
7
|
-
export declare function createDiffusionFitRequest(params: DiffusionFitRequestParams): FitRequestPlan;
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Modes whose load the projection would not describe.
|
|
3
|
-
*
|
|
4
|
-
* `upscale` is an ESRGAN checkpoint, which has no fitter. `world` assembles a
|
|
5
|
-
* TAEHV preview decoder and a seed scene that the fit request has no place for.
|
|
6
|
-
*/
|
|
7
|
-
const UNFITTABLE_MODES = {
|
|
8
|
-
upscale: 'a standalone upscaler load has no fitter',
|
|
9
|
-
world: 'world loads are not representable'
|
|
10
|
-
};
|
|
11
|
-
/**
|
|
12
|
-
* Frames to project a video load at. The fitter reads one frame when nothing
|
|
13
|
-
* is passed, which projects image generation; `run` generates 33. The frame
|
|
14
|
-
* count is a generation parameter, and this assessment sees load config alone.
|
|
15
|
-
*/
|
|
16
|
-
const DEFAULT_VIDEO_FRAMES = 33;
|
|
17
|
-
export function createDiffusionFitRequest(params) {
|
|
18
|
-
const config = (params.modelConfig ?? {});
|
|
19
|
-
const artifacts = params.artifacts ?? {};
|
|
20
|
-
const unfittable = config.mode === undefined ? undefined : UNFITTABLE_MODES[config.mode];
|
|
21
|
-
if (unfittable !== undefined)
|
|
22
|
-
return { supported: false, detail: unfittable };
|
|
23
|
-
// A configured upscaler adds a second resident model the projection does not
|
|
24
|
-
// account for.
|
|
25
|
-
if (config.upscaler !== undefined) {
|
|
26
|
-
return { supported: false, detail: 'a configured upscaler is not part of the projection' };
|
|
27
|
-
}
|
|
28
|
-
const files = {
|
|
29
|
-
model: params.modelPath,
|
|
30
|
-
...(artifacts['clipLModelPath'] !== undefined && { clipL: artifacts['clipLModelPath'] }),
|
|
31
|
-
...(artifacts['clipGModelPath'] !== undefined && { clipG: artifacts['clipGModelPath'] }),
|
|
32
|
-
...(artifacts['t5XxlModelPath'] !== undefined && { t5Xxl: artifacts['t5XxlModelPath'] }),
|
|
33
|
-
...(artifacts['llmModelPath'] !== undefined && { llm: artifacts['llmModelPath'] }),
|
|
34
|
-
...(artifacts['vaeModelPath'] !== undefined && { vae: artifacts['vaeModelPath'] }),
|
|
35
|
-
...(artifacts['uncondModelPath'] !== undefined && {
|
|
36
|
-
uncondModel: artifacts['uncondModelPath']
|
|
37
|
-
}),
|
|
38
|
-
...(artifacts['clipVisionModelPath'] !== undefined && {
|
|
39
|
-
clipVision: artifacts['clipVisionModelPath']
|
|
40
|
-
}),
|
|
41
|
-
...(artifacts['audioVaeModelPath'] !== undefined && {
|
|
42
|
-
audioVae: artifacts['audioVaeModelPath']
|
|
43
|
-
}),
|
|
44
|
-
...(artifacts['embeddingsConnectorsModelPath'] !== undefined && {
|
|
45
|
-
embeddingsConnectors: artifacts['embeddingsConnectorsModelPath']
|
|
46
|
-
})
|
|
47
|
-
};
|
|
48
|
-
// `mode` and `upscaler` are this SDK's own keys; the engine config has no
|
|
49
|
-
// field for either.
|
|
50
|
-
const { mode: _mode, upscaler: _upscaler, ...engineConfig } = config;
|
|
51
|
-
return {
|
|
52
|
-
supported: true,
|
|
53
|
-
probe: {
|
|
54
|
-
engine: 'diffusion-cpp',
|
|
55
|
-
request: {
|
|
56
|
-
files,
|
|
57
|
-
config: engineConfig,
|
|
58
|
-
workload: {
|
|
59
|
-
...(config.vae_tiling !== undefined && { vaeTiling: config.vae_tiling }),
|
|
60
|
-
...(config.mode === 'video' && { videoFrames: DEFAULT_VIDEO_FRAMES })
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
};
|
|
65
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { type CanonicalModelType } from '../../../../schemas/index.js';
|
|
2
|
-
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
3
|
-
export type LlamaLoadKind = 'completion' | 'embedding';
|
|
4
|
-
export declare function llamaLoadKindFor(modelType: CanonicalModelType): LlamaLoadKind | undefined;
|
|
5
|
-
export interface LlamaFitRequestParams {
|
|
6
|
-
loadKind: LlamaLoadKind;
|
|
7
|
-
modelPath: string;
|
|
8
|
-
modelConfig: unknown;
|
|
9
|
-
artifacts?: Record<string, string> | undefined;
|
|
10
|
-
marginBytes?: number | undefined;
|
|
11
|
-
}
|
|
12
|
-
export declare function createLlamaFitRequest(params: LlamaFitRequestParams): FitRequestPlan;
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
import { ModelType } from '../../../../schemas/index.js';
|
|
2
|
-
import { transformLlmConfig } from '../../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
-
import { transformEmbedConfig } from '../../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
|
-
export function llamaLoadKindFor(modelType) {
|
|
5
|
-
if (modelType === ModelType.llamacppCompletion)
|
|
6
|
-
return 'completion';
|
|
7
|
-
if (modelType === ModelType.llamacppEmbedding)
|
|
8
|
-
return 'embedding';
|
|
9
|
-
return undefined;
|
|
10
|
-
}
|
|
11
|
-
export function createLlamaFitRequest(params) {
|
|
12
|
-
const { loadKind } = params;
|
|
13
|
-
if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
|
|
14
|
-
return { supported: false, detail: 'multimodal projection loads are not representable' };
|
|
15
|
-
}
|
|
16
|
-
const modelConfig = (params.modelConfig ?? {});
|
|
17
|
-
// The assessment budgets against this machine's devices, so a load spread
|
|
18
|
-
// over remote GPUs is refused rather than measured against the wrong ones.
|
|
19
|
-
if (modelConfig['rpc-servers'] !== undefined) {
|
|
20
|
-
return { supported: false, detail: 'unsupported load setting: rpc-servers' };
|
|
21
|
-
}
|
|
22
|
-
const request = {
|
|
23
|
-
modelPath: params.modelPath,
|
|
24
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
25
|
-
};
|
|
26
|
-
if (loadKind === 'completion') {
|
|
27
|
-
// The image limit gates images at inference and does not change load
|
|
28
|
-
// memory. The addon's fitter rejects it as an unknown argument.
|
|
29
|
-
const { 'image-max-megapixels': _imageMaxMegapixels, ...fitConfig } = modelConfig;
|
|
30
|
-
const llm = {
|
|
31
|
-
...request,
|
|
32
|
-
config: transformLlmConfig(fitConfig)
|
|
33
|
-
};
|
|
34
|
-
return { supported: true, probe: { engine: 'llm-llamacpp', request: llm } };
|
|
35
|
-
}
|
|
36
|
-
const embed = {
|
|
37
|
-
...request,
|
|
38
|
-
config: transformEmbedConfig(modelConfig)
|
|
39
|
-
};
|
|
40
|
-
return { supported: true, probe: { engine: 'embed-llamacpp', request: embed } };
|
|
41
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
2
|
-
export interface NmtFitRequestParams {
|
|
3
|
-
modelPath: string;
|
|
4
|
-
modelConfig: unknown;
|
|
5
|
-
artifacts?: Record<string, string> | undefined;
|
|
6
|
-
marginBytes?: number | undefined;
|
|
7
|
-
}
|
|
8
|
-
export declare function createNmtFitRequest(params: NmtFitRequestParams): FitRequestPlan;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { nmtConfigFor, nmtFilesFor } from '../../../../plugins/builtin/nmtcpp-translation/load-files.js';
|
|
2
|
-
export function createNmtFitRequest(params) {
|
|
3
|
-
const config = (params.modelConfig ?? {});
|
|
4
|
-
const files = nmtFilesFor(params.modelPath, params.artifacts ?? {});
|
|
5
|
-
const request = {
|
|
6
|
-
files,
|
|
7
|
-
config: nmtConfigFor(config, files.pivotModel),
|
|
8
|
-
...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
|
|
9
|
-
};
|
|
10
|
-
return { supported: true, probe: { engine: 'translation-nmtcpp', request } };
|
|
11
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
2
|
-
export interface OcrFitRequestParams {
|
|
3
|
-
/** The recognizer, which is the load's primary model. */
|
|
4
|
-
modelPath: string;
|
|
5
|
-
modelConfig: unknown;
|
|
6
|
-
artifacts?: Record<string, string> | undefined;
|
|
7
|
-
marginBytes?: number | undefined;
|
|
8
|
-
}
|
|
9
|
-
export declare function createOcrFitRequest(params: OcrFitRequestParams): FitRequestPlan;
|