@qvac/inference 0.18.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -2
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +7 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +26 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +19 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +578 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +45 -0
- package/dist/resources/model-fit/estimators/llm.js +220 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +14 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +16 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +271 -0
- package/dist/schemas/assess-model-fit.js +153 -0
- package/dist/schemas/audio-gen.d.ts +434 -4
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +2 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +2 -2
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +662 -331
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +2 -2
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +1 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2933 -1948
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +0 -1
- package/dist/schemas/model-ops.js +1 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +14 -1
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The least-squares core of the calibration harness
|
|
3
|
+
* (`scripts/calibrate-model-fit.ts`), kept in `src` so it is unit-testable —
|
|
4
|
+
* the harness itself needs real model loads, but the arithmetic that turns
|
|
5
|
+
* measurements into coefficients does not.
|
|
6
|
+
*
|
|
7
|
+
* The model is dictated by how llama.cpp actually allocates: everything is
|
|
8
|
+
* built when the model loads — weights, KV cache, engine overhead, and the
|
|
9
|
+
* context-scaled compute buffers — so a load's RSS delta is
|
|
10
|
+
*
|
|
11
|
+
* persistent = ratio × artifactBytes + kv + fixed + perToken × context
|
|
12
|
+
*
|
|
13
|
+
* The KV term is computed exactly per point and subtracted before fitting, so
|
|
14
|
+
* three unknowns remain: the weight-slack ratio, the fixed overhead, and the
|
|
15
|
+
* per-token slope. Three models at two contexts each give six points for a
|
|
16
|
+
* three-parameter plane; repeats add more.
|
|
17
|
+
*/
|
|
18
|
+
/** One measured load, with the KV cache the engine allocated for it. */
|
|
19
|
+
export interface CalibrationPoint {
|
|
20
|
+
artifactBytes: number;
|
|
21
|
+
contextTokens: number;
|
|
22
|
+
/** KV-cache bytes for this shape and context, exactly determined. */
|
|
23
|
+
kvBytes: number;
|
|
24
|
+
/** RSS delta across the load, after settle. */
|
|
25
|
+
persistentBytes: number;
|
|
26
|
+
}
|
|
27
|
+
/** Coefficients recovered from a set of calibration points. */
|
|
28
|
+
export interface ResidentFit {
|
|
29
|
+
/** Marginal resident bytes per artifact byte. */
|
|
30
|
+
weightRatio: number;
|
|
31
|
+
fixedBytes: number;
|
|
32
|
+
perTokenBytes: number;
|
|
33
|
+
/**
|
|
34
|
+
* Largest amount any point sits above the fitted (and clamped) plane. An
|
|
35
|
+
* upper bound that does not cover an observed point is not an upper bound,
|
|
36
|
+
* so this floors the fixed-overhead upper bound.
|
|
37
|
+
*/
|
|
38
|
+
worstExcessBytes: number;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Fits `persistent − kv = ratio × artifact + perToken × context + fixed` by
|
|
42
|
+
* ordinary least squares.
|
|
43
|
+
*
|
|
44
|
+
* Negative solutions are clamped to zero — a physical coefficient cannot be
|
|
45
|
+
* negative, and a slightly negative fit is measurement noise — but
|
|
46
|
+
* `worstExcessBytes` is computed against the clamped plane, so the clamp can
|
|
47
|
+
* only widen the eventual upper bound, never thin it.
|
|
48
|
+
*
|
|
49
|
+
* @returns The fit, or `undefined` when the points cannot determine three
|
|
50
|
+
* parameters (fewer than three points, or a degenerate design — e.g. a
|
|
51
|
+
* single model or a single context throughout).
|
|
52
|
+
*/
|
|
53
|
+
export declare function fitResidentMemory(points: readonly CalibrationPoint[]): ResidentFit | undefined;
|
|
54
|
+
/** One model's KV growth between its smallest and largest context, computed vs observed. */
|
|
55
|
+
export interface KvGrowth {
|
|
56
|
+
artifactBytes: number;
|
|
57
|
+
kvDeltaBytes: number;
|
|
58
|
+
observedDeltaBytes: number;
|
|
59
|
+
}
|
|
60
|
+
/** Whether the counter behind the measurements sees allocation at all. */
|
|
61
|
+
export interface KvObservation {
|
|
62
|
+
models: readonly KvGrowth[];
|
|
63
|
+
/** Observed over computed growth; `1` when no model has two contexts. */
|
|
64
|
+
ratio: number;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Checks the measurements against the one term the file sizes exactly: the KV
|
|
68
|
+
* cache grows by a known amount between contexts, and compute buffers only add
|
|
69
|
+
* to it. Growth below that means the wrong cache type was subtracted, or a
|
|
70
|
+
* counter that misses allocation. Summed across models, repeats as medians so a
|
|
71
|
+
* cold first load does not read as a shortfall.
|
|
72
|
+
*/
|
|
73
|
+
export declare function kvObservation(points: readonly CalibrationPoint[]): KvObservation;
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The least-squares core of the calibration harness
|
|
3
|
+
* (`scripts/calibrate-model-fit.ts`), kept in `src` so it is unit-testable —
|
|
4
|
+
* the harness itself needs real model loads, but the arithmetic that turns
|
|
5
|
+
* measurements into coefficients does not.
|
|
6
|
+
*
|
|
7
|
+
* The model is dictated by how llama.cpp actually allocates: everything is
|
|
8
|
+
* built when the model loads — weights, KV cache, engine overhead, and the
|
|
9
|
+
* context-scaled compute buffers — so a load's RSS delta is
|
|
10
|
+
*
|
|
11
|
+
* persistent = ratio × artifactBytes + kv + fixed + perToken × context
|
|
12
|
+
*
|
|
13
|
+
* The KV term is computed exactly per point and subtracted before fitting, so
|
|
14
|
+
* three unknowns remain: the weight-slack ratio, the fixed overhead, and the
|
|
15
|
+
* per-token slope. Three models at two contexts each give six points for a
|
|
16
|
+
* three-parameter plane; repeats add more.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Fits `persistent − kv = ratio × artifact + perToken × context + fixed` by
|
|
20
|
+
* ordinary least squares.
|
|
21
|
+
*
|
|
22
|
+
* Negative solutions are clamped to zero — a physical coefficient cannot be
|
|
23
|
+
* negative, and a slightly negative fit is measurement noise — but
|
|
24
|
+
* `worstExcessBytes` is computed against the clamped plane, so the clamp can
|
|
25
|
+
* only widen the eventual upper bound, never thin it.
|
|
26
|
+
*
|
|
27
|
+
* @returns The fit, or `undefined` when the points cannot determine three
|
|
28
|
+
* parameters (fewer than three points, or a degenerate design — e.g. a
|
|
29
|
+
* single model or a single context throughout).
|
|
30
|
+
*/
|
|
31
|
+
export function fitResidentMemory(points) {
|
|
32
|
+
if (points.length < 3)
|
|
33
|
+
return undefined;
|
|
34
|
+
// Normal equations for y = a·x1 + b·x2 + c.
|
|
35
|
+
let s11 = 0;
|
|
36
|
+
let s12 = 0;
|
|
37
|
+
let s1 = 0;
|
|
38
|
+
let s22 = 0;
|
|
39
|
+
let s2 = 0;
|
|
40
|
+
let s1y = 0;
|
|
41
|
+
let s2y = 0;
|
|
42
|
+
let sy = 0;
|
|
43
|
+
const n = points.length;
|
|
44
|
+
for (const point of points) {
|
|
45
|
+
const x1 = point.artifactBytes;
|
|
46
|
+
const x2 = point.contextTokens;
|
|
47
|
+
const y = point.persistentBytes - point.kvBytes;
|
|
48
|
+
s11 += x1 * x1;
|
|
49
|
+
s12 += x1 * x2;
|
|
50
|
+
s1 += x1;
|
|
51
|
+
s22 += x2 * x2;
|
|
52
|
+
s2 += x2;
|
|
53
|
+
s1y += x1 * y;
|
|
54
|
+
s2y += x2 * y;
|
|
55
|
+
sy += y;
|
|
56
|
+
}
|
|
57
|
+
const solution = solve3([
|
|
58
|
+
[s11, s12, s1],
|
|
59
|
+
[s12, s22, s2],
|
|
60
|
+
[s1, s2, n]
|
|
61
|
+
], [s1y, s2y, sy]);
|
|
62
|
+
if (!solution)
|
|
63
|
+
return undefined;
|
|
64
|
+
const weightRatio = Math.max(0, solution[0]);
|
|
65
|
+
const perTokenBytes = Math.max(0, solution[1]);
|
|
66
|
+
const fixedBytes = Math.max(0, solution[2]);
|
|
67
|
+
let worstExcessBytes = 0;
|
|
68
|
+
for (const point of points) {
|
|
69
|
+
const predicted = weightRatio * point.artifactBytes +
|
|
70
|
+
perTokenBytes * point.contextTokens +
|
|
71
|
+
fixedBytes +
|
|
72
|
+
point.kvBytes;
|
|
73
|
+
worstExcessBytes = Math.max(worstExcessBytes, point.persistentBytes - predicted);
|
|
74
|
+
}
|
|
75
|
+
const fit = { weightRatio, fixedBytes, perTokenBytes, worstExcessBytes };
|
|
76
|
+
return fit;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Checks the measurements against the one term the file sizes exactly: the KV
|
|
80
|
+
* cache grows by a known amount between contexts, and compute buffers only add
|
|
81
|
+
* to it. Growth below that means the wrong cache type was subtracted, or a
|
|
82
|
+
* counter that misses allocation. Summed across models, repeats as medians so a
|
|
83
|
+
* cold first load does not read as a shortfall.
|
|
84
|
+
*/
|
|
85
|
+
export function kvObservation(points) {
|
|
86
|
+
const byModel = new Map();
|
|
87
|
+
for (const point of points) {
|
|
88
|
+
const group = byModel.get(point.artifactBytes);
|
|
89
|
+
if (group)
|
|
90
|
+
group.push(point);
|
|
91
|
+
else
|
|
92
|
+
byModel.set(point.artifactBytes, [point]);
|
|
93
|
+
}
|
|
94
|
+
const models = [];
|
|
95
|
+
let kvDeltaTotal = 0;
|
|
96
|
+
let observedDeltaTotal = 0;
|
|
97
|
+
for (const [artifactBytes, group] of byModel) {
|
|
98
|
+
const contexts = [...new Set(group.map((p) => p.contextTokens))].sort((a, b) => a - b);
|
|
99
|
+
if (contexts.length < 2)
|
|
100
|
+
continue;
|
|
101
|
+
const low = group.filter((p) => p.contextTokens === contexts[0]);
|
|
102
|
+
const high = group.filter((p) => p.contextTokens === contexts[contexts.length - 1]);
|
|
103
|
+
const kvDeltaBytes = high[0].kvBytes - low[0].kvBytes;
|
|
104
|
+
const observedDeltaBytes = median(high.map((p) => p.persistentBytes)) - median(low.map((p) => p.persistentBytes));
|
|
105
|
+
models.push({ artifactBytes, kvDeltaBytes, observedDeltaBytes });
|
|
106
|
+
kvDeltaTotal += kvDeltaBytes;
|
|
107
|
+
observedDeltaTotal += observedDeltaBytes;
|
|
108
|
+
}
|
|
109
|
+
return { models, ratio: kvDeltaTotal > 0 ? observedDeltaTotal / kvDeltaTotal : 1 };
|
|
110
|
+
}
|
|
111
|
+
function median(values) {
|
|
112
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
113
|
+
const middle = Math.floor(sorted.length / 2);
|
|
114
|
+
return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Solves a 3×3 linear system by Gaussian elimination with partial pivoting.
|
|
118
|
+
*
|
|
119
|
+
* @returns The solution vector, or `undefined` when the matrix is singular —
|
|
120
|
+
* which for the normal equations means the measurement design cannot
|
|
121
|
+
* separate the three coefficients.
|
|
122
|
+
*/
|
|
123
|
+
function solve3(matrix, rhs) {
|
|
124
|
+
const a = matrix.map((row, i) => [...row, rhs[i]]);
|
|
125
|
+
for (let col = 0; col < 3; col++) {
|
|
126
|
+
let pivot = col;
|
|
127
|
+
for (let row = col + 1; row < 3; row++) {
|
|
128
|
+
if (Math.abs(a[row][col]) > Math.abs(a[pivot][col]))
|
|
129
|
+
pivot = row;
|
|
130
|
+
}
|
|
131
|
+
if (Math.abs(a[pivot][col]) < 1e-9)
|
|
132
|
+
return undefined;
|
|
133
|
+
if (pivot !== col) {
|
|
134
|
+
const swap = a[col];
|
|
135
|
+
a[col] = a[pivot];
|
|
136
|
+
a[pivot] = swap;
|
|
137
|
+
}
|
|
138
|
+
for (let row = col + 1; row < 3; row++) {
|
|
139
|
+
const factor = a[row][col] / a[col][col];
|
|
140
|
+
for (let k = col; k <= 3; k++)
|
|
141
|
+
a[row][k] = a[row][k] - factor * a[col][k];
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const x = [0, 0, 0];
|
|
145
|
+
for (let row = 2; row >= 0; row--) {
|
|
146
|
+
let sum = a[row][3];
|
|
147
|
+
for (let k = row + 1; k < 3; k++)
|
|
148
|
+
sum -= a[row][k] * x[k];
|
|
149
|
+
x[row] = sum / a[row][row];
|
|
150
|
+
}
|
|
151
|
+
return x;
|
|
152
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { CalibrationFixture, ModelFitPlatform, PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Every platform this feature has coefficients for.
|
|
4
|
+
*
|
|
5
|
+
* A platform is added here only once the calibration harness has measured it
|
|
6
|
+
* and a held-out model validated inside the bounds. Everything else assesses as
|
|
7
|
+
* `unknown` — see `METHODOLOGY.md` next to this file.
|
|
8
|
+
*/
|
|
9
|
+
export declare const CALIBRATION: CalibrationFixture;
|
|
10
|
+
/**
|
|
11
|
+
* Looks up the coefficients for a platform.
|
|
12
|
+
*
|
|
13
|
+
* @param platform - Runtime platform-arch pair.
|
|
14
|
+
* @returns The calibration, or `undefined` when the platform is absent or its
|
|
15
|
+
* coefficients are still provisional. Both cases assess as `unknown`.
|
|
16
|
+
*/
|
|
17
|
+
export declare function getPlatformCalibration(platform: ModelFitPlatform): PlatformCalibration | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* Looks up GPU-resident coefficients for a platform on a specific backend.
|
|
20
|
+
*
|
|
21
|
+
* Keyed by backend as well as platform because a `linux-x64` host may run
|
|
22
|
+
* Vulkan, CUDA or ROCm, and their buffers differ — unlike the CPU fixtures,
|
|
23
|
+
* where one platform entry is accepted as covering every backend.
|
|
24
|
+
*
|
|
25
|
+
* @returns `undefined` when that pair has not been measured, which assesses as
|
|
26
|
+
* `unknown` rather than borrowing another backend's coefficients.
|
|
27
|
+
*/
|
|
28
|
+
export declare function getGpuCalibration(platform: ModelFitPlatform, backend: string): PlatformCalibration | undefined;
|
|
29
|
+
/**
|
|
30
|
+
* Looks up integrated-GPU coefficients, spent against the system budget. The
|
|
31
|
+
* gap from the device fixture is large: `win32-x64:vulkan` fits a weight ratio
|
|
32
|
+
* of 1.02 against the card's memory and 2.04 against system RAM.
|
|
33
|
+
*
|
|
34
|
+
* @returns `undefined` when unmeasured, which assesses as `unknown` rather
|
|
35
|
+
* than borrowing the platform's CPU-forced coefficients.
|
|
36
|
+
*/
|
|
37
|
+
export declare function getSharedGpuCalibration(platform: ModelFitPlatform, backend: string): PlatformCalibration | undefined;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { DARWIN_ARM64_CALIBRATION } from './darwin-arm64.js';
|
|
2
|
+
import { DARWIN_X64_CALIBRATION } from './darwin-x64.js';
|
|
3
|
+
import { LINUX_ARM64_CALIBRATION } from './linux-arm64.js';
|
|
4
|
+
import { LINUX_X64_CALIBRATION } from './linux-x64.js';
|
|
5
|
+
import { LINUX_X64_VULKAN_CALIBRATION } from './linux-x64-vulkan.js';
|
|
6
|
+
import { WIN32_X64_CALIBRATION } from './win32-x64.js';
|
|
7
|
+
import { WIN32_X64_VULKAN_CALIBRATION } from './win32-x64-vulkan.js';
|
|
8
|
+
import { WIN32_X64_VULKAN_SHARED_CALIBRATION } from './win32-x64-vulkan-shared.js';
|
|
9
|
+
/**
|
|
10
|
+
* Every platform this feature has coefficients for.
|
|
11
|
+
*
|
|
12
|
+
* A platform is added here only once the calibration harness has measured it
|
|
13
|
+
* and a held-out model validated inside the bounds. Everything else assesses as
|
|
14
|
+
* `unknown` — see `METHODOLOGY.md` next to this file.
|
|
15
|
+
*/
|
|
16
|
+
export const CALIBRATION = {
|
|
17
|
+
schemaVersion: 1,
|
|
18
|
+
platforms: {
|
|
19
|
+
'darwin-arm64': DARWIN_ARM64_CALIBRATION,
|
|
20
|
+
'darwin-x64': DARWIN_X64_CALIBRATION,
|
|
21
|
+
'linux-arm64': LINUX_ARM64_CALIBRATION,
|
|
22
|
+
'linux-x64': LINUX_X64_CALIBRATION,
|
|
23
|
+
'win32-x64': WIN32_X64_CALIBRATION
|
|
24
|
+
},
|
|
25
|
+
gpuPlatforms: {
|
|
26
|
+
'linux-x64:vulkan': LINUX_X64_VULKAN_CALIBRATION,
|
|
27
|
+
'win32-x64:vulkan': WIN32_X64_VULKAN_CALIBRATION
|
|
28
|
+
},
|
|
29
|
+
sharedGpuPlatforms: {
|
|
30
|
+
'win32-x64:vulkan': WIN32_X64_VULKAN_SHARED_CALIBRATION
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Looks up the coefficients for a platform.
|
|
35
|
+
*
|
|
36
|
+
* @param platform - Runtime platform-arch pair.
|
|
37
|
+
* @returns The calibration, or `undefined` when the platform is absent or its
|
|
38
|
+
* coefficients are still provisional. Both cases assess as `unknown`.
|
|
39
|
+
*/
|
|
40
|
+
export function getPlatformCalibration(platform) {
|
|
41
|
+
const calibration = CALIBRATION.platforms[platform];
|
|
42
|
+
if (!calibration || !calibration.validated)
|
|
43
|
+
return undefined;
|
|
44
|
+
return calibration;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Looks up GPU-resident coefficients for a platform on a specific backend.
|
|
48
|
+
*
|
|
49
|
+
* Keyed by backend as well as platform because a `linux-x64` host may run
|
|
50
|
+
* Vulkan, CUDA or ROCm, and their buffers differ — unlike the CPU fixtures,
|
|
51
|
+
* where one platform entry is accepted as covering every backend.
|
|
52
|
+
*
|
|
53
|
+
* @returns `undefined` when that pair has not been measured, which assesses as
|
|
54
|
+
* `unknown` rather than borrowing another backend's coefficients.
|
|
55
|
+
*/
|
|
56
|
+
export function getGpuCalibration(platform, backend) {
|
|
57
|
+
const calibration = CALIBRATION.gpuPlatforms?.[`${platform}:${backend}`];
|
|
58
|
+
if (!calibration || !calibration.validated)
|
|
59
|
+
return undefined;
|
|
60
|
+
return calibration;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Looks up integrated-GPU coefficients, spent against the system budget. The
|
|
64
|
+
* gap from the device fixture is large: `win32-x64:vulkan` fits a weight ratio
|
|
65
|
+
* of 1.02 against the card's memory and 2.04 against system RAM.
|
|
66
|
+
*
|
|
67
|
+
* @returns `undefined` when unmeasured, which assesses as `unknown` rather
|
|
68
|
+
* than borrowing the platform's CPU-forced coefficients.
|
|
69
|
+
*/
|
|
70
|
+
export function getSharedGpuCalibration(platform, backend) {
|
|
71
|
+
const calibration = CALIBRATION.sharedGpuPlatforms?.[`${platform}:${backend}`];
|
|
72
|
+
if (!calibration || !calibration.validated)
|
|
73
|
+
return undefined;
|
|
74
|
+
return calibration;
|
|
75
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* linux-arm64 coefficients.
|
|
4
|
+
*
|
|
5
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
+
* to this file for how the numbers are derived and validated.
|
|
7
|
+
*/
|
|
8
|
+
export declare const LINUX_ARM64_CALIBRATION: PlatformCalibration;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* linux-arm64 coefficients.
|
|
3
|
+
*
|
|
4
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
+
* to this file for how the numbers are derived and validated.
|
|
6
|
+
*/
|
|
7
|
+
export const LINUX_ARM64_CALIBRATION = {
|
|
8
|
+
weightUpperCoeff: 1.088,
|
|
9
|
+
workingPeakBytes: {
|
|
10
|
+
lower: 0,
|
|
11
|
+
upper: 8178893
|
|
12
|
+
},
|
|
13
|
+
fixedOverheadBytes: {
|
|
14
|
+
lower: 108926309,
|
|
15
|
+
upper: 238075911
|
|
16
|
+
},
|
|
17
|
+
computeBufferBytesPerToken: {
|
|
18
|
+
lower: 0,
|
|
19
|
+
upper: 0
|
|
20
|
+
},
|
|
21
|
+
audioWindowBytes: {
|
|
22
|
+
lower: 0,
|
|
23
|
+
upper: 0
|
|
24
|
+
},
|
|
25
|
+
audioStreamingBytes: {
|
|
26
|
+
lower: 0,
|
|
27
|
+
upper: 0
|
|
28
|
+
},
|
|
29
|
+
validated: true,
|
|
30
|
+
notes: [
|
|
31
|
+
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
+
],
|
|
33
|
+
measuredAt: '2026-09-03',
|
|
34
|
+
measuredOn: {
|
|
35
|
+
backend: 'cpu',
|
|
36
|
+
kvElementBytes: 2
|
|
37
|
+
}
|
|
38
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* linux-x64-vulkan coefficients.
|
|
4
|
+
*
|
|
5
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
+
* to this file for how the numbers are derived and validated.
|
|
7
|
+
*/
|
|
8
|
+
export declare const LINUX_X64_VULKAN_CALIBRATION: PlatformCalibration;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* linux-x64-vulkan coefficients.
|
|
3
|
+
*
|
|
4
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
+
* to this file for how the numbers are derived and validated.
|
|
6
|
+
*/
|
|
7
|
+
export const LINUX_X64_VULKAN_CALIBRATION = {
|
|
8
|
+
weightUpperCoeff: 1.019,
|
|
9
|
+
workingPeakBytes: {
|
|
10
|
+
lower: 0,
|
|
11
|
+
upper: 0
|
|
12
|
+
},
|
|
13
|
+
fixedOverheadBytes: {
|
|
14
|
+
lower: 234775431,
|
|
15
|
+
upper: 368894651
|
|
16
|
+
},
|
|
17
|
+
computeBufferBytesPerToken: {
|
|
18
|
+
lower: 0,
|
|
19
|
+
upper: 0
|
|
20
|
+
},
|
|
21
|
+
audioWindowBytes: {
|
|
22
|
+
lower: 0,
|
|
23
|
+
upper: 0
|
|
24
|
+
},
|
|
25
|
+
audioStreamingBytes: {
|
|
26
|
+
lower: 0,
|
|
27
|
+
upper: 0
|
|
28
|
+
},
|
|
29
|
+
validated: true,
|
|
30
|
+
notes: [
|
|
31
|
+
'workingPeakBytes is an unsampled zero, not a measurement: this run polled no device counter during completions. Re-measure before trusting a per-completion budget on this backend'
|
|
32
|
+
],
|
|
33
|
+
measuredAt: '2026-09-03',
|
|
34
|
+
measuredOn: {
|
|
35
|
+
backend: 'vulkan',
|
|
36
|
+
device: 'NVIDIA RTX 4000 SFF Ada Generation',
|
|
37
|
+
kvElementBytes: 1.0625
|
|
38
|
+
}
|
|
39
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* linux-x64 coefficients.
|
|
4
|
+
*
|
|
5
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
+
* to this file for how the numbers are derived and validated.
|
|
7
|
+
*/
|
|
8
|
+
export declare const LINUX_X64_CALIBRATION: PlatformCalibration;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* linux-x64 coefficients.
|
|
3
|
+
*
|
|
4
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
+
* to this file for how the numbers are derived and validated.
|
|
6
|
+
*/
|
|
7
|
+
export const LINUX_X64_CALIBRATION = {
|
|
8
|
+
weightUpperCoeff: 1.01,
|
|
9
|
+
workingPeakBytes: {
|
|
10
|
+
lower: 0,
|
|
11
|
+
upper: 8223130
|
|
12
|
+
},
|
|
13
|
+
fixedOverheadBytes: {
|
|
14
|
+
lower: 0,
|
|
15
|
+
upper: 19194251
|
|
16
|
+
},
|
|
17
|
+
computeBufferBytesPerToken: {
|
|
18
|
+
lower: 927,
|
|
19
|
+
upper: 1391
|
|
20
|
+
},
|
|
21
|
+
audioWindowBytes: {
|
|
22
|
+
lower: 0,
|
|
23
|
+
upper: 0
|
|
24
|
+
},
|
|
25
|
+
audioStreamingBytes: {
|
|
26
|
+
lower: 0,
|
|
27
|
+
upper: 0
|
|
28
|
+
},
|
|
29
|
+
validated: true,
|
|
30
|
+
notes: [
|
|
31
|
+
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
+
],
|
|
33
|
+
measuredAt: '2026-09-03',
|
|
34
|
+
measuredOn: {
|
|
35
|
+
backend: 'cpu',
|
|
36
|
+
kvElementBytes: 2
|
|
37
|
+
}
|
|
38
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* win32-x64-vulkan-shared coefficients.
|
|
4
|
+
*
|
|
5
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
+
* to this file for how the numbers are derived and validated.
|
|
7
|
+
*/
|
|
8
|
+
export declare const WIN32_X64_VULKAN_SHARED_CALIBRATION: PlatformCalibration;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* win32-x64-vulkan-shared coefficients.
|
|
3
|
+
*
|
|
4
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
+
* to this file for how the numbers are derived and validated.
|
|
6
|
+
*/
|
|
7
|
+
export const WIN32_X64_VULKAN_SHARED_CALIBRATION = {
|
|
8
|
+
weightUpperCoeff: 2.044,
|
|
9
|
+
workingPeakBytes: {
|
|
10
|
+
lower: 0,
|
|
11
|
+
upper: 13044941
|
|
12
|
+
},
|
|
13
|
+
fixedOverheadBytes: {
|
|
14
|
+
lower: 239502621,
|
|
15
|
+
upper: 421678470
|
|
16
|
+
},
|
|
17
|
+
computeBufferBytesPerToken: {
|
|
18
|
+
lower: 5932,
|
|
19
|
+
upper: 8898
|
|
20
|
+
},
|
|
21
|
+
audioWindowBytes: {
|
|
22
|
+
lower: 0,
|
|
23
|
+
upper: 0
|
|
24
|
+
},
|
|
25
|
+
audioStreamingBytes: {
|
|
26
|
+
lower: 0,
|
|
27
|
+
upper: 0
|
|
28
|
+
},
|
|
29
|
+
validated: true,
|
|
30
|
+
notes: [
|
|
31
|
+
"measured with 'main-gpu: integrated' and the SDK's default load mode, against process RSS: an integrated GPU allocates out of system RAM, so these coefficients belong to the system-memory basis and not to a device budget"
|
|
32
|
+
],
|
|
33
|
+
measuredAt: '2026-09-03',
|
|
34
|
+
measuredOn: {
|
|
35
|
+
backend: 'vulkan',
|
|
36
|
+
device: 'Intel(R) UHD Graphics 770',
|
|
37
|
+
kvElementBytes: 1.0625
|
|
38
|
+
}
|
|
39
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* win32-x64-vulkan coefficients.
|
|
4
|
+
*
|
|
5
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
+
* to this file for how the numbers are derived and validated.
|
|
7
|
+
*/
|
|
8
|
+
export declare const WIN32_X64_VULKAN_CALIBRATION: PlatformCalibration;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* win32-x64-vulkan coefficients.
|
|
3
|
+
*
|
|
4
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
+
* to this file for how the numbers are derived and validated.
|
|
6
|
+
*/
|
|
7
|
+
export const WIN32_X64_VULKAN_CALIBRATION = {
|
|
8
|
+
weightUpperCoeff: 1.018,
|
|
9
|
+
workingPeakBytes: {
|
|
10
|
+
lower: 0,
|
|
11
|
+
upper: 6606029
|
|
12
|
+
},
|
|
13
|
+
fixedOverheadBytes: {
|
|
14
|
+
lower: 231096560,
|
|
15
|
+
upper: 364959355
|
|
16
|
+
},
|
|
17
|
+
computeBufferBytesPerToken: {
|
|
18
|
+
lower: 0,
|
|
19
|
+
upper: 0
|
|
20
|
+
},
|
|
21
|
+
audioWindowBytes: {
|
|
22
|
+
lower: 0,
|
|
23
|
+
upper: 0
|
|
24
|
+
},
|
|
25
|
+
audioStreamingBytes: {
|
|
26
|
+
lower: 0,
|
|
27
|
+
upper: 0
|
|
28
|
+
},
|
|
29
|
+
validated: true,
|
|
30
|
+
measuredAt: '2026-09-04',
|
|
31
|
+
measuredOn: {
|
|
32
|
+
backend: 'vulkan',
|
|
33
|
+
device: 'NVIDIA RTX 4000 SFF Ada Generation',
|
|
34
|
+
kvElementBytes: 1.0625
|
|
35
|
+
}
|
|
36
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PlatformCalibration } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* win32-x64 coefficients.
|
|
4
|
+
*
|
|
5
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
+
* to this file for how the numbers are derived and validated.
|
|
7
|
+
*/
|
|
8
|
+
export declare const WIN32_X64_CALIBRATION: PlatformCalibration;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* win32-x64 coefficients.
|
|
3
|
+
*
|
|
4
|
+
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
+
* to this file for how the numbers are derived and validated.
|
|
6
|
+
*/
|
|
7
|
+
export const WIN32_X64_CALIBRATION = {
|
|
8
|
+
weightUpperCoeff: 1.01,
|
|
9
|
+
workingPeakBytes: {
|
|
10
|
+
lower: 0,
|
|
11
|
+
upper: 10587341
|
|
12
|
+
},
|
|
13
|
+
fixedOverheadBytes: {
|
|
14
|
+
lower: 55193195,
|
|
15
|
+
upper: 134358525
|
|
16
|
+
},
|
|
17
|
+
computeBufferBytesPerToken: {
|
|
18
|
+
lower: 0,
|
|
19
|
+
upper: 0
|
|
20
|
+
},
|
|
21
|
+
audioWindowBytes: {
|
|
22
|
+
lower: 0,
|
|
23
|
+
upper: 0
|
|
24
|
+
},
|
|
25
|
+
audioStreamingBytes: {
|
|
26
|
+
lower: 0,
|
|
27
|
+
upper: 0
|
|
28
|
+
},
|
|
29
|
+
validated: true,
|
|
30
|
+
notes: [
|
|
31
|
+
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
+
],
|
|
33
|
+
measuredAt: '2026-09-03',
|
|
34
|
+
measuredOn: {
|
|
35
|
+
backend: 'cpu',
|
|
36
|
+
kvElementBytes: 2
|
|
37
|
+
}
|
|
38
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
|
|
2
|
+
import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
|
|
3
|
+
export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
|
|
4
|
+
/**
|
|
5
|
+
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
6
|
+
* metadata alone.
|
|
7
|
+
*
|
|
8
|
+
* Weights come from the artifact size; the KV cache is computed from the
|
|
9
|
+
* transformer shape and the requested context. The estimate is a range, not a
|
|
10
|
+
* number, because two things are genuinely undetermined before load: the
|
|
11
|
+
* backend the engine will pick (which sets the default KV-cache type) and, for
|
|
12
|
+
* some architectures, which blocks hold a full cache.
|
|
13
|
+
*/
|
|
14
|
+
export declare function estimateLlm(input: EstimatorInput): EstimatorResult;
|
|
15
|
+
/** A KV-cache element width range, with the reason it is that range. */
|
|
16
|
+
export interface KvElementWidth {
|
|
17
|
+
bytes: ByteRange;
|
|
18
|
+
assumption: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Picks the KV-cache element width range for this model on this device.
|
|
22
|
+
*
|
|
23
|
+
* Exported because the calibration harness has to subtract the cache the engine
|
|
24
|
+
* *actually* allocated before it can fit the remaining overhead. Hard-coding a
|
|
25
|
+
* width there would skew the fit by roughly 2× on a GPU backend — and because
|
|
26
|
+
* the error scales with context, it would corrupt the per-token slope rather
|
|
27
|
+
* than shifting the intercept. Sharing this rule keeps the two in step.
|
|
28
|
+
*
|
|
29
|
+
* @returns Lower/upper bytes per cache element, and the assumption that choice
|
|
30
|
+
* rests on. Side-effect free so the harness can call it without an
|
|
31
|
+
* assumptions array.
|
|
32
|
+
*/
|
|
33
|
+
export declare function kvElementBytes(facts: GgufFacts, hasGpu: boolean): KvElementWidth;
|
|
34
|
+
/**
|
|
35
|
+
* KV-cache bytes at one fixed element width, clamped to the trained context.
|
|
36
|
+
*
|
|
37
|
+
* Exported for the calibration harness, which must subtract the cache the
|
|
38
|
+
* engine actually allocated using the exact accounting the estimator uses —
|
|
39
|
+
* a hand-rolled copy drifted once already (no sliding-window branch, `ceil`
|
|
40
|
+
* where the estimator bounds with `floor`, no SSM state). A non-degenerate
|
|
41
|
+
* range means part of the layout is engine-owned, so the allocation cannot be
|
|
42
|
+
* known from the file alone and the model is unsuitable for calibration; the
|
|
43
|
+
* harness aborts on it rather than guessing.
|
|
44
|
+
*/
|
|
45
|
+
export declare function kvCacheBytesForWidth(facts: GgufFacts, contextTokens: number, bytesPerElement: number): ByteRange;
|