@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import { type LoadDescription } from './resolve-load-from-stubs.js';
|
|
2
|
-
import type { FitStubOptions, FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
3
|
-
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
|
|
4
|
-
export type LoadFitOutcome = {
|
|
5
|
-
status: 'projected';
|
|
6
|
-
fit: AdvisoryFitOutcome;
|
|
7
|
-
} | {
|
|
8
|
-
status: 'no-stub';
|
|
9
|
-
reason: FitStubUnavailableReason;
|
|
10
|
-
message?: string;
|
|
11
|
-
} | {
|
|
12
|
-
status: 'unsupported-load';
|
|
13
|
-
detail: string;
|
|
14
|
-
};
|
|
15
|
-
export interface LoadFitOptions {
|
|
16
|
-
stub?: FitStubOptions;
|
|
17
|
-
fit?: AdvisoryFitOptions;
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* Projects the load a caller describes, before its weights exist locally.
|
|
21
|
-
*
|
|
22
|
-
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
23
|
-
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
24
|
-
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
25
|
-
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
26
|
-
*/
|
|
27
|
-
export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import { resolveLoadFromStubs } from './resolve-load-from-stubs.js';
|
|
2
|
-
import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
|
|
3
|
-
/**
|
|
4
|
-
* Projects the load a caller describes, before its weights exist locally.
|
|
5
|
-
*
|
|
6
|
-
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
7
|
-
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
8
|
-
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
9
|
-
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
10
|
-
*/
|
|
11
|
-
export async function projectFitFromLoad(load, modelId, options = {}) {
|
|
12
|
-
const resolved = await resolveLoadFromStubs(load, options.stub);
|
|
13
|
-
if (resolved.status !== 'resolved')
|
|
14
|
-
return resolved;
|
|
15
|
-
try {
|
|
16
|
-
const fit = await runAdvisoryFitCheck({
|
|
17
|
-
modelId,
|
|
18
|
-
modelType: load.modelType,
|
|
19
|
-
modelPath: resolved.modelPath,
|
|
20
|
-
modelConfig: resolved.modelConfig,
|
|
21
|
-
transcriptionWorkload: load.transcriptionWorkload,
|
|
22
|
-
ttsWorkload: load.ttsWorkload,
|
|
23
|
-
artifacts: resolved.artifacts,
|
|
24
|
-
// Nothing here assembles a shard set, so no load may claim the fitter
|
|
25
|
-
// is looking at a split model.
|
|
26
|
-
isShardedModel: false
|
|
27
|
-
}, options.fit);
|
|
28
|
-
return { status: 'projected', fit };
|
|
29
|
-
}
|
|
30
|
-
finally {
|
|
31
|
-
await resolved.release();
|
|
32
|
-
}
|
|
33
|
-
}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
-
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
3
|
-
export declare const FLOOR_VERSION = "floor-v1";
|
|
4
|
-
export interface FloorInput {
|
|
5
|
-
profile: ModelResourceProfile;
|
|
6
|
-
workload: ModelFitWorkload;
|
|
7
|
-
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
8
|
-
extraArtifactBytes: number;
|
|
9
|
-
}
|
|
10
|
-
/** A lower bound with no upper bound, and how it was assembled. */
|
|
11
|
-
export interface FloorResult {
|
|
12
|
-
bytes: number;
|
|
13
|
-
reasons: readonly string[];
|
|
14
|
-
assumptions: readonly string[];
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* The smallest resident footprint the catalog facts prove: artifact bytes, plus
|
|
18
|
-
* the KV cache for a llama.cpp model at the cheapest element width the engine
|
|
19
|
-
* can default to.
|
|
20
|
-
*
|
|
21
|
-
* Every term the real load adds on top — weight slack, engine overhead, compute
|
|
22
|
-
* buffers, a completion's working peak — is non-negative, so a floor with all of
|
|
23
|
-
* them at zero is unconditionally valid on any platform and any backend. It can
|
|
24
|
-
* therefore refuse a model (`likely-too-large` when even the floor is over
|
|
25
|
-
* budget) but never confirm one, since how far above the floor the real load
|
|
26
|
-
* lands is unmeasured here.
|
|
27
|
-
*
|
|
28
|
-
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
29
|
-
* fallback wherever no engine measurement is available.
|
|
30
|
-
*/
|
|
31
|
-
export declare function computeFloor(input: FloorInput): FloorResult;
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from './kv-cache.js';
|
|
2
|
-
export const FLOOR_VERSION = 'floor-v1';
|
|
3
|
-
/**
|
|
4
|
-
* Engines whose load builds a KV cache the file sizes exactly, so the floor can
|
|
5
|
-
* count it. Every other engine's working memory is engine-owned and unsized
|
|
6
|
-
* here; the floor for those is the weights alone.
|
|
7
|
-
*/
|
|
8
|
-
const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
|
|
9
|
-
/**
|
|
10
|
-
* The smallest resident footprint the catalog facts prove: artifact bytes, plus
|
|
11
|
-
* the KV cache for a llama.cpp model at the cheapest element width the engine
|
|
12
|
-
* can default to.
|
|
13
|
-
*
|
|
14
|
-
* Every term the real load adds on top — weight slack, engine overhead, compute
|
|
15
|
-
* buffers, a completion's working peak — is non-negative, so a floor with all of
|
|
16
|
-
* them at zero is unconditionally valid on any platform and any backend. It can
|
|
17
|
-
* therefore refuse a model (`likely-too-large` when even the floor is over
|
|
18
|
-
* budget) but never confirm one, since how far above the floor the real load
|
|
19
|
-
* lands is unmeasured here.
|
|
20
|
-
*
|
|
21
|
-
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
22
|
-
* fallback wherever no engine measurement is available.
|
|
23
|
-
*/
|
|
24
|
-
export function computeFloor(input) {
|
|
25
|
-
const { profile, workload, extraArtifactBytes } = input;
|
|
26
|
-
const reasons = [];
|
|
27
|
-
const assumptions = [
|
|
28
|
-
KV_CACHE_ENGINES.has(profile.engine)
|
|
29
|
-
? LLAMA_WEIGHTS_ASSUMPTION
|
|
30
|
-
: 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
|
|
31
|
-
'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
|
|
32
|
-
];
|
|
33
|
-
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
34
|
-
if (extraArtifactBytes > 0) {
|
|
35
|
-
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
36
|
-
}
|
|
37
|
-
if (!KV_CACHE_ENGINES.has(profile.engine)) {
|
|
38
|
-
reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
|
|
39
|
-
return { bytes: artifactBytes, reasons, assumptions };
|
|
40
|
-
}
|
|
41
|
-
if (workload.kind !== 'llm') {
|
|
42
|
-
reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
|
|
43
|
-
return { bytes: artifactBytes, reasons, assumptions };
|
|
44
|
-
}
|
|
45
|
-
const facts = profile.ggufFacts;
|
|
46
|
-
if (!facts) {
|
|
47
|
-
reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
|
|
48
|
-
return { bytes: artifactBytes, reasons, assumptions };
|
|
49
|
-
}
|
|
50
|
-
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
51
|
-
if (contextTokens > facts.contextLength) {
|
|
52
|
-
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
53
|
-
contextTokens = facts.contextLength;
|
|
54
|
-
}
|
|
55
|
-
// The narrowest default the engine can pick on any backend, so the floor
|
|
56
|
-
// holds whether the model lands on a GPU (q8_0 with flash attention) or on
|
|
57
|
-
// the CPU (f16). Architectures that disable flash attention stay f16 anyway.
|
|
58
|
-
const element = narrowestKvElementBytes(facts);
|
|
59
|
-
const kv = kvCacheFloorBytes(facts, contextTokens, element);
|
|
60
|
-
assumptions.push(element < 2
|
|
61
|
-
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
|
-
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
63
|
-
reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
|
|
64
|
-
return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
|
|
65
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import type { GgufFacts } from '../../schemas/model-resource-profile.js';
|
|
2
|
-
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
3
|
-
/**
|
|
4
|
-
* The narrowest KV-cache element width the engine can default to for this
|
|
5
|
-
* model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
|
|
6
|
-
*/
|
|
7
|
-
export declare function narrowestKvElementBytes(facts: GgufFacts): number;
|
|
8
|
-
/**
|
|
9
|
-
* The smallest KV cache the requested context can need, at a fixed element
|
|
10
|
-
* width and clamped to the trained context.
|
|
11
|
-
*
|
|
12
|
-
* Three cases, in order of how much the file actually tells us:
|
|
13
|
-
*
|
|
14
|
-
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
15
|
-
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
16
|
-
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
17
|
-
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
18
|
-
* blocks are which is engine-owned, so the fewest full blocks are counted.
|
|
19
|
-
* 3. **Flat** — every block holds the same cache. A declared sliding window
|
|
20
|
-
* with no per-layer pattern is engine-owned, so every block is counted
|
|
21
|
-
* windowed.
|
|
22
|
-
*/
|
|
23
|
-
export declare function kvCacheFloorBytes(facts: GgufFacts, contextTokens: number, elementBytes: number): number;
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* KV-cache element widths, in bytes per element.
|
|
3
|
-
*
|
|
4
|
-
* `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
|
|
5
|
-
* on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
|
|
6
|
-
* instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318).
|
|
7
|
-
* `q8_0` packs 32 elements into a 34-byte block.
|
|
8
|
-
*/
|
|
9
|
-
const F16_BYTES_PER_ELEMENT = 2;
|
|
10
|
-
const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
|
|
11
|
-
/** Recurrent/SSM state is kept in f32. */
|
|
12
|
-
const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
13
|
-
/**
|
|
14
|
-
* Architectures that disable flash attention, so the GPU `q8_0` KV default
|
|
15
|
-
* never applies and the cache stays `f16` on every backend.
|
|
16
|
-
*/
|
|
17
|
-
function disablesFlashAttention(architecture) {
|
|
18
|
-
return architecture.startsWith('bitnet');
|
|
19
|
-
}
|
|
20
|
-
// Shared with the computed floor so a mixed set carries one weights assumption, not two.
|
|
21
|
-
export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
|
|
22
|
-
/**
|
|
23
|
-
* The narrowest KV-cache element width the engine can default to for this
|
|
24
|
-
* model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
|
|
25
|
-
*/
|
|
26
|
-
export function narrowestKvElementBytes(facts) {
|
|
27
|
-
return disablesFlashAttention(facts.architecture) ? F16_BYTES_PER_ELEMENT : Q8_0_BYTES_PER_ELEMENT;
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* The smallest KV cache the requested context can need, at a fixed element
|
|
31
|
-
* width and clamped to the trained context.
|
|
32
|
-
*
|
|
33
|
-
* Three cases, in order of how much the file actually tells us:
|
|
34
|
-
*
|
|
35
|
-
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
36
|
-
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
37
|
-
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
38
|
-
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
39
|
-
* blocks are which is engine-owned, so the fewest full blocks are counted.
|
|
40
|
-
* 3. **Flat** — every block holds the same cache. A declared sliding window
|
|
41
|
-
* with no per-layer pattern is engine-owned, so every block is counted
|
|
42
|
-
* windowed.
|
|
43
|
-
*/
|
|
44
|
-
export function kvCacheFloorBytes(facts, contextTokens, elementBytes) {
|
|
45
|
-
const tokens = Math.min(contextTokens, facts.contextLength);
|
|
46
|
-
if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
|
|
47
|
-
return layerClassBytes(facts.kvLayerClasses, facts, tokens, elementBytes);
|
|
48
|
-
}
|
|
49
|
-
const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
|
|
50
|
-
if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
|
|
51
|
-
const fullBlocks = Math.floor(facts.blockCount / facts.fullAttentionInterval);
|
|
52
|
-
const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocks);
|
|
53
|
-
return fullBlocks * perBlockPerToken * tokens * elementBytes + ssm;
|
|
54
|
-
}
|
|
55
|
-
if (facts.slidingWindow) {
|
|
56
|
-
const windowedTokens = Math.min(tokens, facts.slidingWindow);
|
|
57
|
-
return facts.blockCount * perBlockPerToken * windowedTokens * elementBytes;
|
|
58
|
-
}
|
|
59
|
-
return facts.blockCount * perBlockPerToken * tokens * elementBytes;
|
|
60
|
-
}
|
|
61
|
-
function layerClassBytes(classes, facts, contextTokens, elementBytes) {
|
|
62
|
-
let total = 0;
|
|
63
|
-
for (const layerClass of classes) {
|
|
64
|
-
const tokens = layerClass.windowed && facts.slidingWindow
|
|
65
|
-
? Math.min(contextTokens, facts.slidingWindow)
|
|
66
|
-
: contextTokens;
|
|
67
|
-
total +=
|
|
68
|
-
layerClass.count *
|
|
69
|
-
layerClass.headCountKv *
|
|
70
|
-
(layerClass.keyLength + layerClass.valueLength) *
|
|
71
|
-
tokens *
|
|
72
|
-
elementBytes;
|
|
73
|
-
}
|
|
74
|
-
return total;
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
|
|
78
|
-
* the SSM state plus its convolution window, per block.
|
|
79
|
-
*/
|
|
80
|
-
function ssmStateBytes(facts, recurrentBlocks) {
|
|
81
|
-
if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
|
|
82
|
-
return 0;
|
|
83
|
-
const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
|
|
84
|
-
return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
|
|
85
|
-
}
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { Logger } from '../../../logging/types.js';
|
|
3
|
-
import type { TtsFitWorkload } from '../../../schemas/assess-model-fit.js';
|
|
4
|
-
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict, TranscriptionFitWorkload } from '../../../schemas/index.js';
|
|
5
|
-
import { runFit as runFitDefault } from './run-fit.js';
|
|
6
|
-
/**
|
|
7
|
-
* `fit` and `does-not-fit` are projections of the load the SDK is about to run.
|
|
8
|
-
* The real loader neither consumes nor verifies the fitted plan.
|
|
9
|
-
*
|
|
10
|
-
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
11
|
-
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
12
|
-
* log line.
|
|
13
|
-
*/
|
|
14
|
-
export type AdvisoryFitVerdict = NativeProbeVerdict;
|
|
15
|
-
export type AdvisoryFitOutcome = NativeProbeFit;
|
|
16
|
-
export interface AdvisoryFitInput {
|
|
17
|
-
transcriptionWorkload?: TranscriptionFitWorkload | undefined;
|
|
18
|
-
ttsWorkload?: TtsFitWorkload | undefined;
|
|
19
|
-
modelId: string;
|
|
20
|
-
modelType: CanonicalModelType;
|
|
21
|
-
modelPath: string;
|
|
22
|
-
modelConfig: unknown;
|
|
23
|
-
artifacts?: Record<string, string> | undefined;
|
|
24
|
-
isShardedModel: boolean;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
28
|
-
* substitute them rather than mocking modules.
|
|
29
|
-
*/
|
|
30
|
-
export interface AdvisoryFitOptions {
|
|
31
|
-
signal?: AbortSignal;
|
|
32
|
-
mobile?: boolean;
|
|
33
|
-
timeoutMs?: number;
|
|
34
|
-
runFit?: typeof runFitDefault;
|
|
35
|
-
logger?: Logger;
|
|
36
|
-
availableSystemBytes?: () => Promise<number | undefined>;
|
|
37
|
-
countsDeviceRows?: () => Promise<boolean>;
|
|
38
|
-
residentModelBytes?: () => Promise<number>;
|
|
39
|
-
/**
|
|
40
|
-
* Bytes to hold back beyond what is resident, for models the caller counts
|
|
41
|
-
* as loaded alongside this one but that this worker does not yet hold.
|
|
42
|
-
*/
|
|
43
|
-
extraResidentBytes?: number;
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Runs the advisory fit check for a load that is about to start, against the
|
|
47
|
-
* fitter belonging to the engine that will run it.
|
|
48
|
-
*
|
|
49
|
-
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
50
|
-
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
51
|
-
* and the caller proceeds with the ordinary load path. This function never
|
|
52
|
-
* throws and never rejects.
|
|
53
|
-
*/
|
|
54
|
-
export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
|
|
@@ -1,248 +0,0 @@
|
|
|
1
|
-
import { getEngineLogger } from '../../../logging/index.js';
|
|
2
|
-
import { classifyFit } from './classify-fit.js';
|
|
3
|
-
import { createFitRequest } from './create-fit-request.js';
|
|
4
|
-
import { runFit as runFitDefault } from './run-fit.js';
|
|
5
|
-
/**
|
|
6
|
-
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
7
|
-
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
8
|
-
* expiry is `unknown` and the load continues.
|
|
9
|
-
*/
|
|
10
|
-
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
11
|
-
/**
|
|
12
|
-
* The base every engine's fitter withholds by default. Made explicit here
|
|
13
|
-
* because the resident-model reserve is added to it: setting a margin at all
|
|
14
|
-
* replaces the engine default, so the base has to travel with it. It is also
|
|
15
|
-
* the headroom `withMachineBudget` leaves on the system budget.
|
|
16
|
-
*/
|
|
17
|
-
const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
18
|
-
/**
|
|
19
|
-
* Names the headroom policy: the fitter withholds
|
|
20
|
-
* `ADVISORY_FIT_BASE_MARGIN_MIB` plus the on-disk bytes of every model resident
|
|
21
|
-
* in this worker, and a `fit` is then judged against the budget
|
|
22
|
-
* `assessModelFit` reports, under that platform's basis and less the
|
|
23
|
-
* `interactive-v1` reserve.
|
|
24
|
-
*/
|
|
25
|
-
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
|
|
26
|
-
const BYTES_PER_MIB = 1024 * 1024;
|
|
27
|
-
/**
|
|
28
|
-
* The mobile runtime flag is imported lazily: that module reaches Bare-only
|
|
29
|
-
* bindings, and resolving it eagerly would make this orchestration untestable
|
|
30
|
-
* outside a Bare runtime.
|
|
31
|
-
*/
|
|
32
|
-
async function resolveMobile(explicit) {
|
|
33
|
-
if (explicit !== undefined)
|
|
34
|
-
return explicit;
|
|
35
|
-
const { isMobile } = await import('../../../runtime/state.js');
|
|
36
|
-
return isMobile();
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Sums the on-disk weight sizes of every model currently registered in this
|
|
40
|
-
* worker. The fallback for a host whose memory sample is unavailable, where
|
|
41
|
-
* models this worker loaded are the one part of the machine's usage it can
|
|
42
|
-
* still account for.
|
|
43
|
-
*
|
|
44
|
-
* Advisory and fail-open like everything else here: any failure to stat a file
|
|
45
|
-
* contributes zero rather than an error.
|
|
46
|
-
*/
|
|
47
|
-
async function defaultResidentModelBytes() {
|
|
48
|
-
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
49
|
-
import('../../../runtime/model-registry.js'),
|
|
50
|
-
import('bare-fs')
|
|
51
|
-
]);
|
|
52
|
-
let bytes = 0;
|
|
53
|
-
for (const id of getAllModelIds()) {
|
|
54
|
-
const info = getModelInfo(id);
|
|
55
|
-
if (info === null)
|
|
56
|
-
continue;
|
|
57
|
-
try {
|
|
58
|
-
const stats = (await fsPromises.stat(info.path));
|
|
59
|
-
bytes += stats.size;
|
|
60
|
-
}
|
|
61
|
-
catch {
|
|
62
|
-
// Unreadable path: contribute nothing rather than fail the check.
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
return bytes;
|
|
66
|
-
}
|
|
67
|
-
const PROVENANCE = {
|
|
68
|
-
basis: 'native-probe',
|
|
69
|
-
estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
|
|
70
|
-
};
|
|
71
|
-
function unknown(reason, message) {
|
|
72
|
-
return message === undefined
|
|
73
|
-
? { ...PROVENANCE, verdict: 'unknown', reason }
|
|
74
|
-
: { ...PROVENANCE, verdict: 'unknown', reason, message };
|
|
75
|
-
}
|
|
76
|
-
function mib(bytes) {
|
|
77
|
-
return `${Math.round(bytes / BYTES_PER_MIB)} MiB`;
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* What the operating system would give back, which no backend reports. Metal
|
|
81
|
-
* answers with its working-set allowance minus this process's own allocations,
|
|
82
|
-
* so another application's resident pages read as free, and the host figure
|
|
83
|
-
* counts little beyond wired and compressed memory.
|
|
84
|
-
*/
|
|
85
|
-
async function defaultAvailableSystemBytes() {
|
|
86
|
-
const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
|
|
87
|
-
import('../../instance.js'),
|
|
88
|
-
import('../assess.js'),
|
|
89
|
-
import('../platform.js')
|
|
90
|
-
]);
|
|
91
|
-
const collector = getResourceCollector();
|
|
92
|
-
if (!collector)
|
|
93
|
-
return undefined;
|
|
94
|
-
const platform = detectPlatform();
|
|
95
|
-
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
96
|
-
const budget = resolveBudget(resources, platform, resolveBasis(platform), []);
|
|
97
|
-
if (budget === undefined)
|
|
98
|
-
return undefined;
|
|
99
|
-
return Math.max(0, budget.availableAfterReserveBytes);
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* Whether the device's allocations come out of the pool the system budget
|
|
103
|
-
* measures. Host bytes always do; device bytes do only where every allocation
|
|
104
|
-
* is system RAM, which is what `boundBySystemMemory` decides.
|
|
105
|
-
*/
|
|
106
|
-
async function deviceBytesCountAgainstSystem() {
|
|
107
|
-
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([
|
|
108
|
-
import('../../instance.js'),
|
|
109
|
-
import('../assess.js'),
|
|
110
|
-
import('../platform.js')
|
|
111
|
-
]);
|
|
112
|
-
const collector = getResourceCollector();
|
|
113
|
-
// Unknown counts them, which errs toward refusing.
|
|
114
|
-
if (!collector)
|
|
115
|
-
return true;
|
|
116
|
-
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
117
|
-
return boundBySystemMemory(resources, detectPlatform());
|
|
118
|
-
}
|
|
119
|
-
/**
|
|
120
|
-
* Peak the projection places on the pool the system budget measures. A
|
|
121
|
-
* discrete card's memory is not that pool, so charging its bytes here would
|
|
122
|
-
* refuse a load the machine can hold.
|
|
123
|
-
*/
|
|
124
|
-
function projectedDemandBytes(outcome, countDevice) {
|
|
125
|
-
const projection = outcome.projection;
|
|
126
|
-
if (projection === undefined)
|
|
127
|
-
return undefined;
|
|
128
|
-
if (projection.deviceBytes === undefined && projection.hostBytes === undefined)
|
|
129
|
-
return undefined;
|
|
130
|
-
return (countDevice ? (projection.deviceBytes ?? 0) : 0) + (projection.hostBytes ?? 0);
|
|
131
|
-
}
|
|
132
|
-
/**
|
|
133
|
-
* Judges a `fit` against the machine. The fitter decides placement and reports
|
|
134
|
-
* what the load costs; only this side knows what is free, so only this side can
|
|
135
|
-
* refuse on capacity.
|
|
136
|
-
*
|
|
137
|
-
* The margin cannot carry this. It is subtracted from each device's own free,
|
|
138
|
-
* and a device whose total sits below the machine's — Metal caps its allowance
|
|
139
|
-
* near 74% of system memory — would be charged the shortfall twice.
|
|
140
|
-
*
|
|
141
|
-
* A projection with no byte totals leaves the engine's verdict standing.
|
|
142
|
-
*/
|
|
143
|
-
function withMachineBudget(outcome, availableBytes, countDevice) {
|
|
144
|
-
if (outcome.verdict !== 'fit' || availableBytes === undefined)
|
|
145
|
-
return outcome;
|
|
146
|
-
const demand = projectedDemandBytes(outcome, countDevice);
|
|
147
|
-
if (demand === undefined)
|
|
148
|
-
return outcome;
|
|
149
|
-
if (demand <= availableBytes)
|
|
150
|
-
return outcome;
|
|
151
|
-
return {
|
|
152
|
-
...outcome,
|
|
153
|
-
verdict: 'does-not-fit',
|
|
154
|
-
reason: 'exceeds-available-memory',
|
|
155
|
-
message: `needs ${mib(demand)} against a ${mib(availableBytes)} budget`
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
/** The device and host totals, for the engines that measure them. */
|
|
159
|
-
function footprint(outcome) {
|
|
160
|
-
const projection = outcome.projection;
|
|
161
|
-
if (projection?.deviceBytes === undefined)
|
|
162
|
-
return '';
|
|
163
|
-
const host = projection.hostBytes === undefined ? '' : `, host ${mib(projection.hostBytes)}`;
|
|
164
|
-
return ` — device ${mib(projection.deviceBytes)}${host}`;
|
|
165
|
-
}
|
|
166
|
-
function report(logger, input, outcome) {
|
|
167
|
-
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
168
|
-
if (outcome.verdict === 'fit') {
|
|
169
|
-
const plan = outcome.plan;
|
|
170
|
-
logger.info(`${prefix} projected to fit${plan === undefined
|
|
171
|
-
? footprint(outcome)
|
|
172
|
-
: ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
|
|
173
|
-
return;
|
|
174
|
-
}
|
|
175
|
-
if (outcome.verdict === 'does-not-fit') {
|
|
176
|
-
logger.warn(`${prefix} projected not to fit${footprint(outcome)}: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`);
|
|
177
|
-
return;
|
|
178
|
-
}
|
|
179
|
-
// `unsupported-load` covers every load this check refuses up front — a model
|
|
180
|
-
// type with no fitter, a companion set that did not resolve — so at `info` it
|
|
181
|
-
// would tag those loads on every start. `debug` for those; `info` where a
|
|
182
|
-
// fitter ran (or should have) and produced no verdict, because there "why was
|
|
183
|
-
// there no evidence" is the most useful thing this check can report.
|
|
184
|
-
const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
|
|
185
|
-
if (outcome.reason === 'unsupported-load') {
|
|
186
|
-
logger.debug(line);
|
|
187
|
-
return;
|
|
188
|
-
}
|
|
189
|
-
logger.info(line);
|
|
190
|
-
}
|
|
191
|
-
/**
|
|
192
|
-
* Runs the advisory fit check for a load that is about to start, against the
|
|
193
|
-
* fitter belonging to the engine that will run it.
|
|
194
|
-
*
|
|
195
|
-
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
196
|
-
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
197
|
-
* and the caller proceeds with the ordinary load path. This function never
|
|
198
|
-
* throws and never rejects.
|
|
199
|
-
*/
|
|
200
|
-
export async function runAdvisoryFitCheck(input, options = {}) {
|
|
201
|
-
let logger = options.logger;
|
|
202
|
-
try {
|
|
203
|
-
logger ??= getEngineLogger();
|
|
204
|
-
const mobile = await resolveMobile(options.mobile);
|
|
205
|
-
const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
|
|
206
|
-
const residentBytes = (await (options.residentModelBytes ?? defaultResidentModelBytes)()) +
|
|
207
|
-
(options.extraResidentBytes ?? 0);
|
|
208
|
-
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
209
|
-
// Always sent, even with a zero reserve, so the base margin has one source
|
|
210
|
-
// and an engine default never decides it.
|
|
211
|
-
const plan = createFitRequest({
|
|
212
|
-
modelType: input.modelType,
|
|
213
|
-
modelPath: input.modelPath,
|
|
214
|
-
modelConfig: input.modelConfig,
|
|
215
|
-
transcriptionWorkload: input.transcriptionWorkload,
|
|
216
|
-
ttsWorkload: input.ttsWorkload,
|
|
217
|
-
artifacts: input.artifacts,
|
|
218
|
-
isShardedModel: input.isShardedModel,
|
|
219
|
-
marginBytes: (ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB) * BYTES_PER_MIB
|
|
220
|
-
});
|
|
221
|
-
if (!plan.supported) {
|
|
222
|
-
const outcome = unknown('unsupported-load', plan.detail);
|
|
223
|
-
report(logger, input, outcome);
|
|
224
|
-
return outcome;
|
|
225
|
-
}
|
|
226
|
-
const result = await (options.runFit ?? runFitDefault)(plan.probe, {
|
|
227
|
-
mobile,
|
|
228
|
-
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
229
|
-
...(options.signal !== undefined && { signal: options.signal })
|
|
230
|
-
});
|
|
231
|
-
const outcome = result.status === 'completed'
|
|
232
|
-
? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
|
|
233
|
-
: unknown(result.reason, result.message);
|
|
234
|
-
report(logger, input, outcome);
|
|
235
|
-
return outcome;
|
|
236
|
-
}
|
|
237
|
-
catch (error) {
|
|
238
|
-
const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
|
|
239
|
-
try {
|
|
240
|
-
if (logger !== undefined)
|
|
241
|
-
report(logger, input, outcome);
|
|
242
|
-
}
|
|
243
|
-
catch {
|
|
244
|
-
// A failing logger must not turn an advisory check into a load failure.
|
|
245
|
-
}
|
|
246
|
-
return outcome;
|
|
247
|
-
}
|
|
248
|
-
}
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Loads the one addon a probe names and asks it for a projection. The child
|
|
3
|
-
* registers no plugins, so the fitter cannot come off the registry here. Only
|
|
4
|
-
* the child reaches this module, and the supervisor spawns it by path rather
|
|
5
|
-
* than importing it, so these specifiers stay out of the graph a host bundles.
|
|
6
|
-
*/
|
|
7
|
-
async function assessFitOf(probe) {
|
|
8
|
-
switch (probe.engine) {
|
|
9
|
-
case 'llm-llamacpp': {
|
|
10
|
-
const { default: LlmLlamacpp } = await import('@qvac/llm-llamacpp');
|
|
11
|
-
return LlmLlamacpp.assessFit(probe.request);
|
|
12
|
-
}
|
|
13
|
-
case 'embed-llamacpp': {
|
|
14
|
-
const { assessFit } = await import('@qvac/embed-llamacpp');
|
|
15
|
-
return assessFit(probe.request);
|
|
16
|
-
}
|
|
17
|
-
case 'asr-ggml': {
|
|
18
|
-
const { default: ASRGgml } = await import('@qvac/asr-ggml');
|
|
19
|
-
return ASRGgml.assessFit(probe.request);
|
|
20
|
-
}
|
|
21
|
-
case 'bci-whispercpp': {
|
|
22
|
-
const { assessFit } = await import('@qvac/bci-whispercpp');
|
|
23
|
-
return assessFit(probe.request);
|
|
24
|
-
}
|
|
25
|
-
case 'tts-ggml': {
|
|
26
|
-
const { default: TTSGgml } = await import('@qvac/tts-ggml');
|
|
27
|
-
return TTSGgml.assessFit(probe.request);
|
|
28
|
-
}
|
|
29
|
-
case 'audiogen-ggml': {
|
|
30
|
-
const { assessFit } = await import('@qvac/audiogen-ggml');
|
|
31
|
-
return assessFit(probe.request);
|
|
32
|
-
}
|
|
33
|
-
case 'diffusion-cpp': {
|
|
34
|
-
const { assessFit } = await import('@qvac/diffusion-cpp');
|
|
35
|
-
return assessFit(probe.request);
|
|
36
|
-
}
|
|
37
|
-
case 'ocr-ggml': {
|
|
38
|
-
const { assessFit } = await import('@qvac/ocr-ggml');
|
|
39
|
-
return assessFit(probe.request);
|
|
40
|
-
}
|
|
41
|
-
case 'translation-nmtcpp': {
|
|
42
|
-
const { default: TranslationNmtcpp } = await import('@qvac/translation-nmtcpp');
|
|
43
|
-
return TranslationNmtcpp.assessFit(probe.request);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
export async function callChildEngineFit(probe) {
|
|
48
|
-
return { engine: probe.engine, result: await assessFitOf(probe) };
|
|
49
|
-
}
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { NativeProbeFit } from '../../../schemas/index.js';
|
|
2
|
-
import type { FitProbeResult } from './engine-fit.js';
|
|
3
|
-
export declare function classifyFit(probe: FitProbeResult, provenance: {
|
|
4
|
-
basis: 'native-probe';
|
|
5
|
-
estimatorVersion: string;
|
|
6
|
-
}): NativeProbeFit;
|