@qvac/inference 0.18.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -2
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +7 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +26 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +19 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +578 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +45 -0
- package/dist/resources/model-fit/estimators/llm.js +220 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +14 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +16 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +271 -0
- package/dist/schemas/assess-model-fit.js +153 -0
- package/dist/schemas/audio-gen.d.ts +434 -4
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +2 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +2 -2
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +662 -331
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +2 -2
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +1 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2933 -1948
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +0 -1
- package/dist/schemas/model-ops.js +1 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +14 -1
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
export const LLM_ESTIMATOR_VERSION = 'llm-v1';
|
|
2
|
+
/**
|
|
3
|
+
* KV-cache element widths, in bytes per element.
|
|
4
|
+
*
|
|
5
|
+
* `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
|
|
6
|
+
* on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
|
|
7
|
+
* instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318),
|
|
8
|
+
* which is why the bound is a range rather than a number. `q8_0` packs 32
|
|
9
|
+
* elements into a 34-byte block.
|
|
10
|
+
*/
|
|
11
|
+
const F16_BYTES_PER_ELEMENT = 2;
|
|
12
|
+
const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
|
|
13
|
+
/** Recurrent/SSM state is kept in f32. */
|
|
14
|
+
const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
15
|
+
/**
|
|
16
|
+
* Architectures that disable flash attention, so the GPU `q8_0` KV default
|
|
17
|
+
* never applies and the cache stays `f16` on every backend.
|
|
18
|
+
*/
|
|
19
|
+
function disablesFlashAttention(architecture) {
|
|
20
|
+
return architecture.startsWith('bitnet');
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
24
|
+
* metadata alone.
|
|
25
|
+
*
|
|
26
|
+
* Weights come from the artifact size; the KV cache is computed from the
|
|
27
|
+
* transformer shape and the requested context. The estimate is a range, not a
|
|
28
|
+
* number, because two things are genuinely undetermined before load: the
|
|
29
|
+
* backend the engine will pick (which sets the default KV-cache type) and, for
|
|
30
|
+
* some architectures, which blocks hold a full cache.
|
|
31
|
+
*/
|
|
32
|
+
export function estimateLlm(input) {
|
|
33
|
+
const { profile, workload, calibration, extraArtifactBytes, hasGpu } = input;
|
|
34
|
+
if (workload.kind !== 'llm') {
|
|
35
|
+
return {
|
|
36
|
+
kind: 'unknown',
|
|
37
|
+
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
38
|
+
reasons: [`workload kind '${workload.kind}' is not supported by ${LLM_ESTIMATOR_VERSION}`]
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
const facts = profile.ggufFacts;
|
|
42
|
+
if (!facts) {
|
|
43
|
+
return {
|
|
44
|
+
kind: 'unknown',
|
|
45
|
+
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
46
|
+
reasons: ['no GGUF metadata for this model in the catalog, so the KV cache cannot be sized']
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
const assumptions = [];
|
|
50
|
+
const reasons = [];
|
|
51
|
+
// Weights: mapped at file size by default, so artifact bytes are the floor.
|
|
52
|
+
// The upper coefficient covers the allocator's copy-on-write and alignment
|
|
53
|
+
// slack measured during calibration.
|
|
54
|
+
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
55
|
+
assumptions.push('weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM');
|
|
56
|
+
if (extraArtifactBytes > 0) {
|
|
57
|
+
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
58
|
+
}
|
|
59
|
+
// Context: never more than the model was trained for — llama.cpp clamps.
|
|
60
|
+
let contextTokens = workload.contextTokens;
|
|
61
|
+
if (contextTokens > facts.contextLength) {
|
|
62
|
+
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
63
|
+
contextTokens = facts.contextLength;
|
|
64
|
+
}
|
|
65
|
+
const element = kvElementBytes(facts, hasGpu);
|
|
66
|
+
assumptions.push(element.assumption);
|
|
67
|
+
const kv = kvCacheBytes(facts, contextTokens, element.bytes, assumptions, reasons);
|
|
68
|
+
// llama.cpp builds the context at load — KV cache, engine overhead, compute
|
|
69
|
+
// buffers — so all of it is resident for the model's lifetime. Parking any of
|
|
70
|
+
// it in `working` would let `sequential` count only the largest. What a
|
|
71
|
+
// completion adds on top is released afterwards, so that part is `working`.
|
|
72
|
+
const persistent = {
|
|
73
|
+
lower: Math.ceil(artifactBytes +
|
|
74
|
+
kv.lower +
|
|
75
|
+
calibration.fixedOverheadBytes.lower +
|
|
76
|
+
calibration.computeBufferBytesPerToken.lower * contextTokens),
|
|
77
|
+
upper: Math.ceil(artifactBytes * calibration.weightUpperCoeff +
|
|
78
|
+
kv.upper +
|
|
79
|
+
calibration.fixedOverheadBytes.upper +
|
|
80
|
+
calibration.computeBufferBytesPerToken.upper * contextTokens)
|
|
81
|
+
};
|
|
82
|
+
assumptions.push('the KV cache, engine overhead and compute buffers count as resident for the model’s whole lifetime; llama.cpp allocates them when the model loads, not per operation');
|
|
83
|
+
if (calibration.workingPeakBytes && calibration.workingPeakBytes.upper > 0) {
|
|
84
|
+
assumptions.push('one operation is assumed in flight per model; the working peak is what a single completion was measured to add on top of the resident cost');
|
|
85
|
+
}
|
|
86
|
+
assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not expressible in a workload and would change these numbers');
|
|
87
|
+
return {
|
|
88
|
+
kind: 'estimate',
|
|
89
|
+
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
90
|
+
persistent,
|
|
91
|
+
working: {
|
|
92
|
+
lower: calibration.workingPeakBytes?.lower ?? 0,
|
|
93
|
+
upper: calibration.workingPeakBytes?.upper ?? 0
|
|
94
|
+
},
|
|
95
|
+
reasons,
|
|
96
|
+
assumptions
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Picks the KV-cache element width range for this model on this device.
|
|
101
|
+
*
|
|
102
|
+
* Exported because the calibration harness has to subtract the cache the engine
|
|
103
|
+
* *actually* allocated before it can fit the remaining overhead. Hard-coding a
|
|
104
|
+
* width there would skew the fit by roughly 2× on a GPU backend — and because
|
|
105
|
+
* the error scales with context, it would corrupt the per-token slope rather
|
|
106
|
+
* than shifting the intercept. Sharing this rule keeps the two in step.
|
|
107
|
+
*
|
|
108
|
+
* @returns Lower/upper bytes per cache element, and the assumption that choice
|
|
109
|
+
* rests on. Side-effect free so the harness can call it without an
|
|
110
|
+
* assumptions array.
|
|
111
|
+
*/
|
|
112
|
+
export function kvElementBytes(facts, hasGpu) {
|
|
113
|
+
if (disablesFlashAttention(facts.architecture)) {
|
|
114
|
+
return {
|
|
115
|
+
bytes: { lower: F16_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
|
|
116
|
+
assumption: `${facts.architecture} loads with flash attention off, so the KV cache stays f16 on every backend`
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
if (hasGpu) {
|
|
120
|
+
return {
|
|
121
|
+
bytes: { lower: Q8_0_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
|
|
122
|
+
assumption: 'a GPU is present, so the engine may default the KV cache to q8_0 (lower bound) or keep f16 on a CPU or OpenCL backend (upper bound)'
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
bytes: { lower: F16_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
|
|
127
|
+
assumption: 'no GPU reported, so the CPU f16 KV-cache default applies'
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* KV-cache bytes at one fixed element width, clamped to the trained context.
|
|
132
|
+
*
|
|
133
|
+
* Exported for the calibration harness, which must subtract the cache the
|
|
134
|
+
* engine actually allocated using the exact accounting the estimator uses —
|
|
135
|
+
* a hand-rolled copy drifted once already (no sliding-window branch, `ceil`
|
|
136
|
+
* where the estimator bounds with `floor`, no SSM state). A non-degenerate
|
|
137
|
+
* range means part of the layout is engine-owned, so the allocation cannot be
|
|
138
|
+
* known from the file alone and the model is unsuitable for calibration; the
|
|
139
|
+
* harness aborts on it rather than guessing.
|
|
140
|
+
*/
|
|
141
|
+
export function kvCacheBytesForWidth(facts, contextTokens, bytesPerElement) {
|
|
142
|
+
const tokens = Math.min(contextTokens, facts.contextLength);
|
|
143
|
+
return kvCacheBytes(facts, tokens, { lower: bytesPerElement, upper: bytesPerElement }, [], []);
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Sizes the KV cache for the requested context.
|
|
147
|
+
*
|
|
148
|
+
* Three cases, in order of how much the file actually tells us:
|
|
149
|
+
*
|
|
150
|
+
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
151
|
+
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
152
|
+
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
153
|
+
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
154
|
+
* blocks are which is engine-owned, so the count is bounded, not fixed.
|
|
155
|
+
* 3. **Flat** — every block holds the same cache. When the file declares a
|
|
156
|
+
* sliding window but no per-layer pattern, the pattern lives in the engine:
|
|
157
|
+
* the bound then spans "every block windowed" to "every block full", which is
|
|
158
|
+
* wide on purpose.
|
|
159
|
+
*/
|
|
160
|
+
function kvCacheBytes(facts, contextTokens, elementBytes, assumptions, reasons) {
|
|
161
|
+
if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
|
|
162
|
+
reasons.push('KV cache summed per layer class from the file’s per-block attention metadata');
|
|
163
|
+
return {
|
|
164
|
+
lower: layerClassBytes(facts.kvLayerClasses, facts, contextTokens, elementBytes.lower),
|
|
165
|
+
upper: layerClassBytes(facts.kvLayerClasses, facts, contextTokens, elementBytes.upper)
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
|
|
169
|
+
if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
|
|
170
|
+
const interval = facts.fullAttentionInterval;
|
|
171
|
+
const fullBlocksLower = Math.floor(facts.blockCount / interval);
|
|
172
|
+
const fullBlocksUpper = Math.ceil(facts.blockCount / interval);
|
|
173
|
+
const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocksLower);
|
|
174
|
+
assumptions.push(`${facts.architecture} keeps full attention every ${interval} blocks; the remaining blocks hold a fixed-size recurrent state instead of a KV cache, and which blocks are which is engine-owned`);
|
|
175
|
+
reasons.push(`KV cache sized for ${fullBlocksLower}–${fullBlocksUpper} of ${facts.blockCount} blocks (hybrid attention)`);
|
|
176
|
+
return {
|
|
177
|
+
lower: fullBlocksLower * perBlockPerToken * contextTokens * elementBytes.lower + ssm,
|
|
178
|
+
upper: fullBlocksUpper * perBlockPerToken * contextTokens * elementBytes.upper + ssm
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
if (facts.slidingWindow) {
|
|
182
|
+
const windowedTokens = Math.min(contextTokens, facts.slidingWindow);
|
|
183
|
+
assumptions.push(`${facts.architecture} uses sliding-window attention with a ${facts.slidingWindow}-token window, but the file does not say which blocks are windowed; the bound spans every block windowed to every block full`);
|
|
184
|
+
reasons.push('sliding-window layer pattern is engine-owned, so the KV bound is wide');
|
|
185
|
+
return {
|
|
186
|
+
lower: facts.blockCount * perBlockPerToken * windowedTokens * elementBytes.lower,
|
|
187
|
+
upper: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.upper
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
reasons.push(`KV cache sized for all ${facts.blockCount} blocks at ${contextTokens} tokens`);
|
|
191
|
+
return {
|
|
192
|
+
lower: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.lower,
|
|
193
|
+
upper: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.upper
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
function layerClassBytes(classes, facts, contextTokens, elementBytes) {
|
|
197
|
+
let total = 0;
|
|
198
|
+
for (const layerClass of classes) {
|
|
199
|
+
const tokens = layerClass.windowed && facts.slidingWindow
|
|
200
|
+
? Math.min(contextTokens, facts.slidingWindow)
|
|
201
|
+
: contextTokens;
|
|
202
|
+
total +=
|
|
203
|
+
layerClass.count *
|
|
204
|
+
layerClass.headCountKv *
|
|
205
|
+
(layerClass.keyLength + layerClass.valueLength) *
|
|
206
|
+
tokens *
|
|
207
|
+
elementBytes;
|
|
208
|
+
}
|
|
209
|
+
return total;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
|
|
213
|
+
* the SSM state plus its convolution window, per block.
|
|
214
|
+
*/
|
|
215
|
+
function ssmStateBytes(facts, recurrentBlocks) {
|
|
216
|
+
if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
|
|
217
|
+
return 0;
|
|
218
|
+
const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
|
|
219
|
+
return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
|
|
220
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EstimatorInput, EstimatorResult } from '../types.js';
|
|
2
|
+
export declare const WHISPER_ESTIMATOR_VERSION = "whisper-v1";
|
|
3
|
+
/**
|
|
4
|
+
* Estimates memory for a whisper.cpp transcription model.
|
|
5
|
+
*
|
|
6
|
+
* Whisper's encoder work is fixed by its 30 s window rather than by a growing
|
|
7
|
+
* cache, so the shape is simpler than the LLM case: resident weights plus a
|
|
8
|
+
* working peak that scales with the window the caller feeds it.
|
|
9
|
+
*/
|
|
10
|
+
export declare function estimateWhisper(input: EstimatorInput): EstimatorResult;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
export const WHISPER_ESTIMATOR_VERSION = 'whisper-v1';
|
|
2
|
+
/** Whisper's own default window, and therefore the calibration anchor. */
|
|
3
|
+
const CALIBRATION_WINDOW_MS = 30_000;
|
|
4
|
+
/**
|
|
5
|
+
* Estimates memory for a whisper.cpp transcription model.
|
|
6
|
+
*
|
|
7
|
+
* Whisper's encoder work is fixed by its 30 s window rather than by a growing
|
|
8
|
+
* cache, so the shape is simpler than the LLM case: resident weights plus a
|
|
9
|
+
* working peak that scales with the window the caller feeds it.
|
|
10
|
+
*/
|
|
11
|
+
export function estimateWhisper(input) {
|
|
12
|
+
const { profile, workload, calibration, extraArtifactBytes } = input;
|
|
13
|
+
if (workload.kind !== 'audio') {
|
|
14
|
+
return {
|
|
15
|
+
kind: 'unknown',
|
|
16
|
+
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
17
|
+
reasons: [`workload kind '${workload.kind}' is not supported by ${WHISPER_ESTIMATOR_VERSION}`]
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
// A zero here is an unmeasured placeholder, not a measurement: the harness
|
|
21
|
+
// writes `{ lower: 0, upper: 0 }` until its whisper pass has run. Consuming
|
|
22
|
+
// it would return a confident estimate whose entire audio working memory is
|
|
23
|
+
// zero — the same failure shape as a mis-subtracted KV cache.
|
|
24
|
+
if (calibration.audioWindowBytes.upper <= 0) {
|
|
25
|
+
return {
|
|
26
|
+
kind: 'unknown',
|
|
27
|
+
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
28
|
+
reasons: [
|
|
29
|
+
'the audio window coefficient for this platform has not been measured, so an audio estimate cannot be defended'
|
|
30
|
+
]
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (workload.streaming && calibration.audioStreamingBytes.upper <= 0) {
|
|
34
|
+
return {
|
|
35
|
+
kind: 'unknown',
|
|
36
|
+
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
37
|
+
reasons: [
|
|
38
|
+
'the streaming-session coefficient for this platform has not been measured, so a streaming audio estimate cannot be defended'
|
|
39
|
+
]
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const assumptions = [];
|
|
43
|
+
const reasons = [];
|
|
44
|
+
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
45
|
+
const persistent = {
|
|
46
|
+
lower: artifactBytes,
|
|
47
|
+
upper: Math.ceil(artifactBytes * calibration.weightUpperCoeff)
|
|
48
|
+
};
|
|
49
|
+
if (extraArtifactBytes > 0) {
|
|
50
|
+
assumptions.push('companion artifacts passed in `artifacts` (e.g. a VAD model) are counted at full size');
|
|
51
|
+
}
|
|
52
|
+
const windows = workload.windowMs / CALIBRATION_WINDOW_MS;
|
|
53
|
+
const batch = workload.batch ?? 1;
|
|
54
|
+
const streamingLower = workload.streaming ? calibration.audioStreamingBytes.lower : 0;
|
|
55
|
+
const streamingUpper = workload.streaming ? calibration.audioStreamingBytes.upper : 0;
|
|
56
|
+
const working = {
|
|
57
|
+
lower: calibration.fixedOverheadBytes.lower +
|
|
58
|
+
calibration.audioWindowBytes.lower * windows * batch +
|
|
59
|
+
streamingLower,
|
|
60
|
+
upper: calibration.fixedOverheadBytes.upper +
|
|
61
|
+
calibration.audioWindowBytes.upper * windows * batch +
|
|
62
|
+
streamingUpper
|
|
63
|
+
};
|
|
64
|
+
reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms${batch > 1 ? ` × ${batch} concurrent windows` : ''}`);
|
|
65
|
+
assumptions.push('the audio window scales working memory linearly from the engine’s 30 s default; longer windows are chunked by the engine rather than held whole');
|
|
66
|
+
if (workload.streaming) {
|
|
67
|
+
assumptions.push('a streaming session is assumed to keep one calibrated stream buffer resident');
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
kind: 'estimate',
|
|
71
|
+
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
72
|
+
persistent,
|
|
73
|
+
working: { lower: Math.ceil(working.lower), upper: Math.ceil(working.upper) },
|
|
74
|
+
reasons,
|
|
75
|
+
assumptions
|
|
76
|
+
};
|
|
77
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
3
|
+
/** A memory bound. `lower` is the optimistic end, `upper` the conservative one. */
|
|
4
|
+
export interface ByteRange {
|
|
5
|
+
lower: number;
|
|
6
|
+
upper: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Platforms a calibration fixture can cover. A platform absent from the shipped
|
|
10
|
+
* fixtures assesses as `unknown`: an uncalibrated formula is not evidence.
|
|
11
|
+
*/
|
|
12
|
+
export type ModelFitPlatform = 'darwin-arm64' | 'darwin-x64' | 'linux-arm64' | 'linux-x64' | 'win32-x64' | 'win32-arm64' | 'android-arm64' | 'ios-arm64';
|
|
13
|
+
/**
|
|
14
|
+
* Measured coefficients for one platform. Derived by the calibration harness
|
|
15
|
+
* (`scripts/calibrate-model-fit.ts`) from real loads, then committed.
|
|
16
|
+
*
|
|
17
|
+
* @property weightUpperCoeff - Multiplier on `artifactBytes` for resident
|
|
18
|
+
* weights at the conservative end. The lower end is always 1.0: llama.cpp
|
|
19
|
+
* maps weights, so the file size is the floor.
|
|
20
|
+
* @property fixedOverheadBytes - Runtime cost that does not scale with the
|
|
21
|
+
* workload: engine context, backend buffers, tokenizer.
|
|
22
|
+
* @property computeBufferBytesPerToken - Compute/graph buffers that scale with
|
|
23
|
+
* the context the caller asks for.
|
|
24
|
+
* @property audioWindowBytes - Whisper-family working memory for one 30 s
|
|
25
|
+
* window, the engine's own default window.
|
|
26
|
+
* @property audioStreamingBytes - Extra resident cost of a streaming session.
|
|
27
|
+
* @property validated - False while coefficients are provisional. An
|
|
28
|
+
* unvalidated platform assesses as `unknown`, exactly like a missing one.
|
|
29
|
+
* @property measuredOn - What the harness observed while measuring. Recorded
|
|
30
|
+
* because these coefficients are keyed by platform alone, while the buffers
|
|
31
|
+
* they measure are allocated by the backend: a `linux-x64` fixture measured on
|
|
32
|
+
* CUDA is being applied to CPU-only hosts too. Stating each fixture's scope
|
|
33
|
+
* keeps that visible, and means the key could later gain a backend dimension
|
|
34
|
+
* without re-measuring anything. Absent on a placeholder, which measured
|
|
35
|
+
* nothing.
|
|
36
|
+
*/
|
|
37
|
+
export interface PlatformCalibration {
|
|
38
|
+
weightUpperCoeff: number;
|
|
39
|
+
fixedOverheadBytes: ByteRange;
|
|
40
|
+
computeBufferBytesPerToken: ByteRange;
|
|
41
|
+
/**
|
|
42
|
+
* Transient peak one operation adds on top of the resident cost — a few MiB
|
|
43
|
+
* on most platforms, 73 MiB on darwin-x64. Optional: fixtures measured
|
|
44
|
+
* before the harness sampled generation contribute nothing, as they do now.
|
|
45
|
+
*/
|
|
46
|
+
workingPeakBytes?: ByteRange;
|
|
47
|
+
audioWindowBytes: ByteRange;
|
|
48
|
+
audioStreamingBytes: ByteRange;
|
|
49
|
+
validated: boolean;
|
|
50
|
+
measuredAt?: string;
|
|
51
|
+
measuredOn?: CalibrationProvenance;
|
|
52
|
+
notes?: readonly string[];
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* The measurement conditions behind one set of coefficients.
|
|
56
|
+
*
|
|
57
|
+
* @property backend - Best-effort label for the backend in play during the run,
|
|
58
|
+
* inferred from the GPU drivers the resource collector reported: `metal`,
|
|
59
|
+
* `vulkan`, `cuda`, `rocm`, or `cpu`.
|
|
60
|
+
* @property device - GPU name, when one was reported.
|
|
61
|
+
* @property kvElementBytes - Bytes per KV-cache element the residuals were
|
|
62
|
+
* computed against. The single most useful number here: subtract the wrong
|
|
63
|
+
* width and the fit is wrong, so recording it makes a bad run auditable after
|
|
64
|
+
* the fact.
|
|
65
|
+
*/
|
|
66
|
+
export interface CalibrationProvenance {
|
|
67
|
+
backend: string;
|
|
68
|
+
device?: string;
|
|
69
|
+
kvElementBytes: number;
|
|
70
|
+
}
|
|
71
|
+
export interface CalibrationFixture {
|
|
72
|
+
schemaVersion: 1;
|
|
73
|
+
platforms: Partial<Record<ModelFitPlatform, PlatformCalibration>>;
|
|
74
|
+
/**
|
|
75
|
+
* GPU-resident coefficients, keyed `<platform>:<backend>`. Separate from
|
|
76
|
+
* `platforms` because those describe CPU-resident execution in system RAM,
|
|
77
|
+
* and because one platform can run several backends whose buffers differ.
|
|
78
|
+
*/
|
|
79
|
+
gpuPlatforms?: Readonly<Record<string, PlatformCalibration>>;
|
|
80
|
+
/**
|
|
81
|
+
* Integrated-GPU coefficients, keyed the same way. A third map because the
|
|
82
|
+
* buffers are the backend's, as in `gpuPlatforms`, but come out of system
|
|
83
|
+
* RAM, as in `platforms`. Measured by the harness's `--igpu` pass.
|
|
84
|
+
*/
|
|
85
|
+
sharedGpuPlatforms?: Readonly<Record<string, PlatformCalibration>>;
|
|
86
|
+
}
|
|
87
|
+
/** What an estimator is handed for one candidate. */
|
|
88
|
+
export interface EstimatorInput {
|
|
89
|
+
profile: ModelResourceProfile;
|
|
90
|
+
workload: ModelFitWorkload;
|
|
91
|
+
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
92
|
+
extraArtifactBytes: number;
|
|
93
|
+
calibration: PlatformCalibration;
|
|
94
|
+
/**
|
|
95
|
+
* Whether the device reports a GPU. Used only to decide which KV-cache type
|
|
96
|
+
* the engine would default to — never as a memory budget, since GPU memory
|
|
97
|
+
* metrics are `unverified`-scoped by design.
|
|
98
|
+
*/
|
|
99
|
+
hasGpu: boolean;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* An estimate, or a refusal to estimate.
|
|
103
|
+
*
|
|
104
|
+
* `persistent` stays resident for the model's whole lifetime; `working` is the
|
|
105
|
+
* temporary peak of one operation. Aggregation treats them differently, which
|
|
106
|
+
* is what makes `sequential` cheaper than `concurrent`.
|
|
107
|
+
*/
|
|
108
|
+
export type EstimatorResult = {
|
|
109
|
+
kind: 'estimate';
|
|
110
|
+
estimatorVersion: string;
|
|
111
|
+
persistent: ByteRange;
|
|
112
|
+
working: ByteRange;
|
|
113
|
+
reasons: readonly string[];
|
|
114
|
+
assumptions: readonly string[];
|
|
115
|
+
} | {
|
|
116
|
+
kind: 'unknown';
|
|
117
|
+
estimatorVersion: string;
|
|
118
|
+
reasons: readonly string[];
|
|
119
|
+
assumptions?: readonly string[];
|
|
120
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import type { ResourceCollectorDependencies } from '
|
|
1
|
+
import type { ResourceCollectorDependencies } from './types.js';
|
|
2
2
|
export declare const nativeResourceCollectorDependencies: ResourceCollectorDependencies;
|
package/dist/resources/native.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os from 'bare-os';
|
|
1
2
|
import { randomUUID } from 'bare-crypto';
|
|
2
3
|
// Load independently so one unavailable native addon cannot disable the other collector.
|
|
3
4
|
const [cpuModule, gpuModule] = await Promise.allSettled([
|
|
@@ -16,6 +17,7 @@ const gpuTypes = gpuModule.status === 'fulfilled'
|
|
|
16
17
|
export const nativeResourceCollectorDependencies = {
|
|
17
18
|
cpuArchitectures,
|
|
18
19
|
gpuTypes,
|
|
20
|
+
platform: os.platform(),
|
|
19
21
|
createCPUInfo() {
|
|
20
22
|
if (cpuModule.status !== 'fulfilled')
|
|
21
23
|
return undefined;
|
|
@@ -31,5 +33,17 @@ export const nativeResourceCollectorDependencies = {
|
|
|
31
33
|
},
|
|
32
34
|
now() {
|
|
33
35
|
return Date.now();
|
|
36
|
+
},
|
|
37
|
+
sampleProcessMemory() {
|
|
38
|
+
// A throw here is a sampling failure, not a platform gap: let it reach the
|
|
39
|
+
// collector so the metric reports `failed` rather than `unavailable`.
|
|
40
|
+
const usage = os.memoryUsage();
|
|
41
|
+
// No platform exposes the per-process allowance yet. On iOS this is
|
|
42
|
+
// `os_proc_available_memory()` — the limit jetsam actually enforces — and
|
|
43
|
+
// it needs a native source before the metric can report a value.
|
|
44
|
+
return {
|
|
45
|
+
usedBytes: usage && usage.rss > 0 ? usage.rss : undefined,
|
|
46
|
+
availableBytes: undefined
|
|
47
|
+
};
|
|
34
48
|
}
|
|
35
49
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ResourceMetric, ResourceProvenance } from '
|
|
1
|
+
import type { ResourceMetric, ResourceProvenance } from './types.js';
|
|
2
2
|
export declare function unavailableMetric<T>(reason?: string): ResourceMetric<T>;
|
|
3
3
|
export declare function unverifiedMetric<T>(reason?: string): ResourceMetric<T>;
|
|
4
4
|
export declare function failedMetric<T>(reason: string): ResourceMetric<T>;
|
|
@@ -50,13 +50,29 @@ export interface GPUInfoContext {
|
|
|
50
50
|
sample(index: number): NativeGPUUsage;
|
|
51
51
|
destroy(): void;
|
|
52
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* The process's own memory position, as opposed to the system-wide view.
|
|
55
|
+
*
|
|
56
|
+
* @property usedBytes - The process's resident footprint (RSS).
|
|
57
|
+
* @property availableBytes - How much more this process may allocate before
|
|
58
|
+
* the OS intervenes. Only some platforms can state this — on iOS it is
|
|
59
|
+
* `os_proc_available_memory()`, the number jetsam actually enforces.
|
|
60
|
+
* `undefined` where no such per-process limit is exposed.
|
|
61
|
+
*/
|
|
62
|
+
export interface ProcessMemorySample {
|
|
63
|
+
usedBytes: number | undefined;
|
|
64
|
+
availableBytes: number | undefined;
|
|
65
|
+
}
|
|
53
66
|
export interface ResourceCollectorDependencies {
|
|
54
67
|
cpuArchitectures: readonly number[];
|
|
55
68
|
gpuTypes: readonly number[];
|
|
69
|
+
/** Runtime platform, used to grade what a GPU memory reading can mean. */
|
|
70
|
+
platform: string;
|
|
56
71
|
createCPUInfo(): CPUInfoContext | undefined;
|
|
57
72
|
createGPUInfo(): GPUInfoContext | undefined;
|
|
58
73
|
createGPUId(): string;
|
|
59
74
|
now(): number;
|
|
75
|
+
sampleProcessMemory(): ProcessMemorySample;
|
|
60
76
|
}
|
|
61
77
|
export interface SystemResourceCollector {
|
|
62
78
|
getCapabilities(): SystemResourceCapabilities;
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { createDisposableScope } from '
|
|
2
|
-
export type { DisposableScope } from '
|
|
3
|
-
export type { RequestContext, RequestKind, RequestState, RequestLogContext } from '
|
|
4
|
-
export { createRequestRegistry, getRequestRegistry, withRequestContext } from '
|
|
5
|
-
export type { BeginOpts, CancelByModelId, CancelByRequestId, CancelTarget, ConcurrencyPolicy, ManagedRequestContext, RequestOutcome, RequestRegistry } from '
|
|
1
|
+
export { createDisposableScope } from './disposable-scope.js';
|
|
2
|
+
export type { DisposableScope } from './disposable-scope.js';
|
|
3
|
+
export type { RequestContext, RequestKind, RequestState, RequestLogContext } from './request-context.js';
|
|
4
|
+
export { createRequestRegistry, getRequestRegistry, withRequestContext } from './request-context.js';
|
|
5
|
+
export type { BeginOpts, CancelByModelId, CancelByRequestId, CancelTarget, ConcurrencyPolicy, ManagedRequestContext, RequestOutcome, RequestRegistry } from './request-registry.js';
|
package/dist/runtime/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { createDisposableScope } from '
|
|
2
|
-
export { createRequestRegistry, getRequestRegistry, withRequestContext } from '
|
|
1
|
+
export { createDisposableScope } from './disposable-scope.js';
|
|
2
|
+
export { createRequestRegistry, getRequestRegistry, withRequestContext } from './request-context.js';
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { initEnv } from '../runtime/env.js';
|
|
1
|
+
import { initEnv } from './env.js';
|
|
3
2
|
import { closeAllRagInstances } from '../rag/index.js';
|
|
4
3
|
import { cleanupDownloads } from '../handlers/load-model/download-manager.js';
|
|
5
|
-
import { unloadAllModels } from '
|
|
6
|
-
import { closeRegistryClient } from '
|
|
7
|
-
import { clearAllLoggingStreams, startLogBuffering, stopLogBufferingWithTimeout } from '
|
|
4
|
+
import { unloadAllModels } from './model-registry.js';
|
|
5
|
+
import { closeRegistryClient } from './registry-client.js';
|
|
6
|
+
import { clearAllLoggingStreams, startLogBuffering, stopLogBufferingWithTimeout } from './logging-stream-registry.js';
|
|
8
7
|
import { clearAllAddonLoggers, getEngineLogger, LOG_ID, ALL_LOG_ID } from '../logging/index.js';
|
|
9
8
|
import { clearPlugins } from '../plugins/index.js';
|
|
10
|
-
import { acquireCacheLock, releaseCacheLock } from '
|
|
9
|
+
import { acquireCacheLock, releaseCacheLock } from './cache-lock.js';
|
|
11
10
|
import { nativeResourceCollectorDependencies } from '../resources/native.js';
|
|
12
11
|
import { destroyResourceCollector, initializeResourceCollector } from '../resources/instance.js';
|
|
13
12
|
// The host application owns the Bare runtime lifecycle: this module sets up the
|
|
@@ -59,7 +58,6 @@ async function runCleanup() {
|
|
|
59
58
|
destroyResourceCollector();
|
|
60
59
|
clearRegistries();
|
|
61
60
|
await Promise.allSettled([
|
|
62
|
-
destroySwarm(),
|
|
63
61
|
closeAllRagInstances(),
|
|
64
62
|
cleanupDownloads(),
|
|
65
63
|
unloadAllModels(),
|
|
@@ -19,11 +19,6 @@ export interface AnyModel {
|
|
|
19
19
|
reload?(config: unknown): Promise<void>;
|
|
20
20
|
addon?: AddonInterface;
|
|
21
21
|
}
|
|
22
|
-
interface DelegateOptions {
|
|
23
|
-
providerPublicKey: string;
|
|
24
|
-
timeout?: number | undefined;
|
|
25
|
-
healthCheckTimeout?: number | undefined;
|
|
26
|
-
}
|
|
27
22
|
interface LocalOptions {
|
|
28
23
|
model: AnyModel;
|
|
29
24
|
path: string;
|
|
@@ -34,11 +29,6 @@ interface LocalOptions {
|
|
|
34
29
|
}
|
|
35
30
|
export type ModelEntry = {
|
|
36
31
|
id: string;
|
|
37
|
-
isDelegated: true;
|
|
38
|
-
delegated: DelegateOptions;
|
|
39
|
-
} | {
|
|
40
|
-
id: string;
|
|
41
|
-
isDelegated: false;
|
|
42
32
|
local: LocalOptions;
|
|
43
33
|
};
|
|
44
34
|
export declare function registerModel(id: string, options: {
|
|
@@ -47,10 +37,6 @@ export declare function registerModel(id: string, options: {
|
|
|
47
37
|
config: unknown;
|
|
48
38
|
modelType: CanonicalModelType;
|
|
49
39
|
name?: string | undefined;
|
|
50
|
-
} | {
|
|
51
|
-
providerPublicKey: string;
|
|
52
|
-
timeout?: number;
|
|
53
|
-
healthCheckTimeout?: number;
|
|
54
40
|
}): void;
|
|
55
41
|
export declare function getModelEntry(id: string): ModelEntry | null;
|
|
56
42
|
export declare function getModel(id: string): AnyModel;
|