@qvac/inference 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +50 -25
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* linux-x64-vulkan coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const LINUX_X64_VULKAN_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.019,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 0
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 234775431,
|
|
15
|
-
upper: 368894651
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 0,
|
|
19
|
-
upper: 0
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
notes: [
|
|
31
|
-
'workingPeakBytes is an unsampled zero, not a measurement: this run polled no device counter during completions. Re-measure before trusting a per-completion budget on this backend'
|
|
32
|
-
],
|
|
33
|
-
measuredAt: '2026-09-03',
|
|
34
|
-
measuredOn: {
|
|
35
|
-
backend: 'vulkan',
|
|
36
|
-
device: 'NVIDIA RTX 4000 SFF Ada Generation',
|
|
37
|
-
kvElementBytes: 1.0625
|
|
38
|
-
}
|
|
39
|
-
};
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* linux-x64 coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const LINUX_X64_CALIBRATION: PlatformCalibration;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* linux-x64 coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const LINUX_X64_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.01,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 8223130
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 0,
|
|
15
|
-
upper: 19194251
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 927,
|
|
19
|
-
upper: 1391
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
notes: [
|
|
31
|
-
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
-
],
|
|
33
|
-
measuredAt: '2026-09-03',
|
|
34
|
-
measuredOn: {
|
|
35
|
-
backend: 'cpu',
|
|
36
|
-
kvElementBytes: 2
|
|
37
|
-
}
|
|
38
|
-
};
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* win32-x64-vulkan-shared coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const WIN32_X64_VULKAN_SHARED_CALIBRATION: PlatformCalibration;
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* win32-x64-vulkan-shared coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const WIN32_X64_VULKAN_SHARED_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 2.044,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 13044941
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 239502621,
|
|
15
|
-
upper: 421678470
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 5932,
|
|
19
|
-
upper: 8898
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
notes: [
|
|
31
|
-
"measured with 'main-gpu: integrated' and the SDK's default load mode, against process RSS: an integrated GPU allocates out of system RAM, so these coefficients belong to the system-memory basis and not to a device budget"
|
|
32
|
-
],
|
|
33
|
-
measuredAt: '2026-09-03',
|
|
34
|
-
measuredOn: {
|
|
35
|
-
backend: 'vulkan',
|
|
36
|
-
device: 'Intel(R) UHD Graphics 770',
|
|
37
|
-
kvElementBytes: 1.0625
|
|
38
|
-
}
|
|
39
|
-
};
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* win32-x64-vulkan coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const WIN32_X64_VULKAN_CALIBRATION: PlatformCalibration;
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* win32-x64-vulkan coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const WIN32_X64_VULKAN_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.018,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 6606029
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 231096560,
|
|
15
|
-
upper: 364959355
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 0,
|
|
19
|
-
upper: 0
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
measuredAt: '2026-09-04',
|
|
31
|
-
measuredOn: {
|
|
32
|
-
backend: 'vulkan',
|
|
33
|
-
device: 'NVIDIA RTX 4000 SFF Ada Generation',
|
|
34
|
-
kvElementBytes: 1.0625
|
|
35
|
-
}
|
|
36
|
-
};
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* win32-x64 coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const WIN32_X64_CALIBRATION: PlatformCalibration;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* win32-x64 coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const WIN32_X64_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.01,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 10587341
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 55193195,
|
|
15
|
-
upper: 134358525
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 0,
|
|
19
|
-
upper: 0
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
notes: [
|
|
31
|
-
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
-
],
|
|
33
|
-
measuredAt: '2026-09-03',
|
|
34
|
-
measuredOn: {
|
|
35
|
-
backend: 'cpu',
|
|
36
|
-
kvElementBytes: 2
|
|
37
|
-
}
|
|
38
|
-
};
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
|
|
2
|
-
import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
|
|
3
|
-
export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
|
|
4
|
-
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
5
|
-
/**
|
|
6
|
-
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
7
|
-
* metadata alone.
|
|
8
|
-
*
|
|
9
|
-
* Weights come from the artifact size; the KV cache is computed from the
|
|
10
|
-
* transformer shape and the requested context. The estimate is a range, not a
|
|
11
|
-
* number, because two things are genuinely undetermined before load: the
|
|
12
|
-
* backend the engine will pick (which sets the default KV-cache type) and, for
|
|
13
|
-
* some architectures, which blocks hold a full cache.
|
|
14
|
-
*/
|
|
15
|
-
export declare function estimateLlm(input: EstimatorInput): EstimatorResult;
|
|
16
|
-
/** A KV-cache element width range, with the reason it is that range. */
|
|
17
|
-
export interface KvElementWidth {
|
|
18
|
-
bytes: ByteRange;
|
|
19
|
-
assumption: string;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Picks the KV-cache element width range for this model on this device.
|
|
23
|
-
*
|
|
24
|
-
* Exported because the calibration harness has to subtract the cache the engine
|
|
25
|
-
* *actually* allocated before it can fit the remaining overhead. Hard-coding a
|
|
26
|
-
* width there would skew the fit by roughly 2× on a GPU backend — and because
|
|
27
|
-
* the error scales with context, it would corrupt the per-token slope rather
|
|
28
|
-
* than shifting the intercept. Sharing this rule keeps the two in step.
|
|
29
|
-
*
|
|
30
|
-
* @returns Lower/upper bytes per cache element, and the assumption that choice
|
|
31
|
-
* rests on. Side-effect free so the harness can call it without an
|
|
32
|
-
* assumptions array.
|
|
33
|
-
*/
|
|
34
|
-
export declare function kvElementBytes(facts: GgufFacts, hasGpu: boolean): KvElementWidth;
|
|
35
|
-
/**
|
|
36
|
-
* KV-cache bytes at one fixed element width, clamped to the trained context.
|
|
37
|
-
*
|
|
38
|
-
* Exported for the calibration harness, which must subtract the cache the
|
|
39
|
-
* engine actually allocated using the exact accounting the estimator uses —
|
|
40
|
-
* a hand-rolled copy drifted once already (no sliding-window branch, `ceil`
|
|
41
|
-
* where the estimator bounds with `floor`, no SSM state). A non-degenerate
|
|
42
|
-
* range means part of the layout is engine-owned, so the allocation cannot be
|
|
43
|
-
* known from the file alone and the model is unsuitable for calibration; the
|
|
44
|
-
* harness aborts on it rather than guessing.
|
|
45
|
-
*/
|
|
46
|
-
export declare function kvCacheBytesForWidth(facts: GgufFacts, contextTokens: number, bytesPerElement: number): ByteRange;
|
|
@@ -1,222 +0,0 @@
|
|
|
1
|
-
export const LLM_ESTIMATOR_VERSION = 'llm-v1';
|
|
2
|
-
/**
|
|
3
|
-
* KV-cache element widths, in bytes per element.
|
|
4
|
-
*
|
|
5
|
-
* `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
|
|
6
|
-
* on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
|
|
7
|
-
* instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318),
|
|
8
|
-
* which is why the bound is a range rather than a number. `q8_0` packs 32
|
|
9
|
-
* elements into a 34-byte block.
|
|
10
|
-
*/
|
|
11
|
-
const F16_BYTES_PER_ELEMENT = 2;
|
|
12
|
-
const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
|
|
13
|
-
/** Recurrent/SSM state is kept in f32. */
|
|
14
|
-
const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
15
|
-
/**
|
|
16
|
-
* Architectures that disable flash attention, so the GPU `q8_0` KV default
|
|
17
|
-
* never applies and the cache stays `f16` on every backend.
|
|
18
|
-
*/
|
|
19
|
-
function disablesFlashAttention(architecture) {
|
|
20
|
-
return architecture.startsWith('bitnet');
|
|
21
|
-
}
|
|
22
|
-
// Shared with the computed floor so a mixed set carries one weights assumption, not two.
|
|
23
|
-
export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
|
|
24
|
-
/**
|
|
25
|
-
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
26
|
-
* metadata alone.
|
|
27
|
-
*
|
|
28
|
-
* Weights come from the artifact size; the KV cache is computed from the
|
|
29
|
-
* transformer shape and the requested context. The estimate is a range, not a
|
|
30
|
-
* number, because two things are genuinely undetermined before load: the
|
|
31
|
-
* backend the engine will pick (which sets the default KV-cache type) and, for
|
|
32
|
-
* some architectures, which blocks hold a full cache.
|
|
33
|
-
*/
|
|
34
|
-
export function estimateLlm(input) {
|
|
35
|
-
const { profile, workload, calibration, extraArtifactBytes, hasGpu } = input;
|
|
36
|
-
if (workload.kind !== 'llm') {
|
|
37
|
-
return {
|
|
38
|
-
kind: 'unknown',
|
|
39
|
-
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
40
|
-
reasons: [`workload kind '${workload.kind}' is not supported by ${LLM_ESTIMATOR_VERSION}`]
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
const facts = profile.ggufFacts;
|
|
44
|
-
if (!facts) {
|
|
45
|
-
return {
|
|
46
|
-
kind: 'unknown',
|
|
47
|
-
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
48
|
-
reasons: ['no GGUF metadata for this model in the catalog, so the KV cache cannot be sized']
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
const assumptions = [];
|
|
52
|
-
const reasons = [];
|
|
53
|
-
// Weights: mapped at file size by default, so artifact bytes are the floor.
|
|
54
|
-
// The upper coefficient covers the allocator's copy-on-write and alignment
|
|
55
|
-
// slack measured during calibration.
|
|
56
|
-
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
57
|
-
assumptions.push(LLAMA_WEIGHTS_ASSUMPTION);
|
|
58
|
-
if (extraArtifactBytes > 0) {
|
|
59
|
-
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
60
|
-
}
|
|
61
|
-
// Context: never more than the model was trained for — llama.cpp clamps.
|
|
62
|
-
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
63
|
-
if (contextTokens > facts.contextLength) {
|
|
64
|
-
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
65
|
-
contextTokens = facts.contextLength;
|
|
66
|
-
}
|
|
67
|
-
const element = kvElementBytes(facts, hasGpu);
|
|
68
|
-
assumptions.push(element.assumption);
|
|
69
|
-
const kv = kvCacheBytes(facts, contextTokens, element.bytes, assumptions, reasons);
|
|
70
|
-
// llama.cpp builds the context at load — KV cache, engine overhead, compute
|
|
71
|
-
// buffers — so all of it is resident for the model's lifetime. Parking any of
|
|
72
|
-
// it in `working` would let `sequential` count only the largest. What a
|
|
73
|
-
// completion adds on top is released afterwards, so that part is `working`.
|
|
74
|
-
const persistent = {
|
|
75
|
-
lower: Math.ceil(artifactBytes +
|
|
76
|
-
kv.lower +
|
|
77
|
-
calibration.fixedOverheadBytes.lower +
|
|
78
|
-
calibration.computeBufferBytesPerToken.lower * contextTokens),
|
|
79
|
-
upper: Math.ceil(artifactBytes * calibration.weightUpperCoeff +
|
|
80
|
-
kv.upper +
|
|
81
|
-
calibration.fixedOverheadBytes.upper +
|
|
82
|
-
calibration.computeBufferBytesPerToken.upper * contextTokens)
|
|
83
|
-
};
|
|
84
|
-
assumptions.push('the KV cache, engine overhead and compute buffers count as resident for the model’s whole lifetime; llama.cpp allocates them when the model loads, not per operation');
|
|
85
|
-
if (calibration.workingPeakBytes && calibration.workingPeakBytes.upper > 0) {
|
|
86
|
-
assumptions.push('one operation is assumed in flight per model; the working peak is what a single completion was measured to add on top of the resident cost');
|
|
87
|
-
}
|
|
88
|
-
assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
89
|
-
return {
|
|
90
|
-
kind: 'estimate',
|
|
91
|
-
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
92
|
-
persistent,
|
|
93
|
-
working: {
|
|
94
|
-
lower: calibration.workingPeakBytes?.lower ?? 0,
|
|
95
|
-
upper: calibration.workingPeakBytes?.upper ?? 0
|
|
96
|
-
},
|
|
97
|
-
reasons,
|
|
98
|
-
assumptions
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* Picks the KV-cache element width range for this model on this device.
|
|
103
|
-
*
|
|
104
|
-
* Exported because the calibration harness has to subtract the cache the engine
|
|
105
|
-
* *actually* allocated before it can fit the remaining overhead. Hard-coding a
|
|
106
|
-
* width there would skew the fit by roughly 2× on a GPU backend — and because
|
|
107
|
-
* the error scales with context, it would corrupt the per-token slope rather
|
|
108
|
-
* than shifting the intercept. Sharing this rule keeps the two in step.
|
|
109
|
-
*
|
|
110
|
-
* @returns Lower/upper bytes per cache element, and the assumption that choice
|
|
111
|
-
* rests on. Side-effect free so the harness can call it without an
|
|
112
|
-
* assumptions array.
|
|
113
|
-
*/
|
|
114
|
-
export function kvElementBytes(facts, hasGpu) {
|
|
115
|
-
if (disablesFlashAttention(facts.architecture)) {
|
|
116
|
-
return {
|
|
117
|
-
bytes: { lower: F16_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
|
|
118
|
-
assumption: `${facts.architecture} loads with flash attention off, so the KV cache stays f16 on every backend`
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
if (hasGpu) {
|
|
122
|
-
return {
|
|
123
|
-
bytes: { lower: Q8_0_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
|
|
124
|
-
assumption: 'a GPU is present, so the engine may default the KV cache to q8_0 (lower bound) or keep f16 on a CPU or OpenCL backend (upper bound)'
|
|
125
|
-
};
|
|
126
|
-
}
|
|
127
|
-
return {
|
|
128
|
-
bytes: { lower: F16_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
|
|
129
|
-
assumption: 'no GPU reported, so the CPU f16 KV-cache default applies'
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
/**
|
|
133
|
-
* KV-cache bytes at one fixed element width, clamped to the trained context.
|
|
134
|
-
*
|
|
135
|
-
* Exported for the calibration harness, which must subtract the cache the
|
|
136
|
-
* engine actually allocated using the exact accounting the estimator uses —
|
|
137
|
-
* a hand-rolled copy drifted once already (no sliding-window branch, `ceil`
|
|
138
|
-
* where the estimator bounds with `floor`, no SSM state). A non-degenerate
|
|
139
|
-
* range means part of the layout is engine-owned, so the allocation cannot be
|
|
140
|
-
* known from the file alone and the model is unsuitable for calibration; the
|
|
141
|
-
* harness aborts on it rather than guessing.
|
|
142
|
-
*/
|
|
143
|
-
export function kvCacheBytesForWidth(facts, contextTokens, bytesPerElement) {
|
|
144
|
-
const tokens = Math.min(contextTokens, facts.contextLength);
|
|
145
|
-
return kvCacheBytes(facts, tokens, { lower: bytesPerElement, upper: bytesPerElement }, [], []);
|
|
146
|
-
}
|
|
147
|
-
/**
|
|
148
|
-
* Sizes the KV cache for the requested context.
|
|
149
|
-
*
|
|
150
|
-
* Three cases, in order of how much the file actually tells us:
|
|
151
|
-
*
|
|
152
|
-
* 1. **Per-layer classes** — the file describes attention per block, so the
|
|
153
|
-
* cache is summed exactly, with sliding-window blocks capped at their window.
|
|
154
|
-
* 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
|
|
155
|
-
* blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
|
|
156
|
-
* blocks are which is engine-owned, so the count is bounded, not fixed.
|
|
157
|
-
* 3. **Flat** — every block holds the same cache. When the file declares a
|
|
158
|
-
* sliding window but no per-layer pattern, the pattern lives in the engine:
|
|
159
|
-
* the bound then spans "every block windowed" to "every block full", which is
|
|
160
|
-
* wide on purpose.
|
|
161
|
-
*/
|
|
162
|
-
function kvCacheBytes(facts, contextTokens, elementBytes, assumptions, reasons) {
|
|
163
|
-
if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
|
|
164
|
-
reasons.push('KV cache summed per layer class from the file’s per-block attention metadata');
|
|
165
|
-
return {
|
|
166
|
-
lower: layerClassBytes(facts.kvLayerClasses, facts, contextTokens, elementBytes.lower),
|
|
167
|
-
upper: layerClassBytes(facts.kvLayerClasses, facts, contextTokens, elementBytes.upper)
|
|
168
|
-
};
|
|
169
|
-
}
|
|
170
|
-
const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
|
|
171
|
-
if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
|
|
172
|
-
const interval = facts.fullAttentionInterval;
|
|
173
|
-
const fullBlocksLower = Math.floor(facts.blockCount / interval);
|
|
174
|
-
const fullBlocksUpper = Math.ceil(facts.blockCount / interval);
|
|
175
|
-
const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocksLower);
|
|
176
|
-
assumptions.push(`${facts.architecture} keeps full attention every ${interval} blocks; the remaining blocks hold a fixed-size recurrent state instead of a KV cache, and which blocks are which is engine-owned`);
|
|
177
|
-
reasons.push(`KV cache sized for ${fullBlocksLower}–${fullBlocksUpper} of ${facts.blockCount} blocks (hybrid attention)`);
|
|
178
|
-
return {
|
|
179
|
-
lower: fullBlocksLower * perBlockPerToken * contextTokens * elementBytes.lower + ssm,
|
|
180
|
-
upper: fullBlocksUpper * perBlockPerToken * contextTokens * elementBytes.upper + ssm
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
if (facts.slidingWindow) {
|
|
184
|
-
const windowedTokens = Math.min(contextTokens, facts.slidingWindow);
|
|
185
|
-
assumptions.push(`${facts.architecture} uses sliding-window attention with a ${facts.slidingWindow}-token window, but the file does not say which blocks are windowed; the bound spans every block windowed to every block full`);
|
|
186
|
-
reasons.push('sliding-window layer pattern is engine-owned, so the KV bound is wide');
|
|
187
|
-
return {
|
|
188
|
-
lower: facts.blockCount * perBlockPerToken * windowedTokens * elementBytes.lower,
|
|
189
|
-
upper: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.upper
|
|
190
|
-
};
|
|
191
|
-
}
|
|
192
|
-
reasons.push(`KV cache sized for all ${facts.blockCount} blocks at ${contextTokens} tokens`);
|
|
193
|
-
return {
|
|
194
|
-
lower: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.lower,
|
|
195
|
-
upper: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.upper
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
function layerClassBytes(classes, facts, contextTokens, elementBytes) {
|
|
199
|
-
let total = 0;
|
|
200
|
-
for (const layerClass of classes) {
|
|
201
|
-
const tokens = layerClass.windowed && facts.slidingWindow
|
|
202
|
-
? Math.min(contextTokens, facts.slidingWindow)
|
|
203
|
-
: contextTokens;
|
|
204
|
-
total +=
|
|
205
|
-
layerClass.count *
|
|
206
|
-
layerClass.headCountKv *
|
|
207
|
-
(layerClass.keyLength + layerClass.valueLength) *
|
|
208
|
-
tokens *
|
|
209
|
-
elementBytes;
|
|
210
|
-
}
|
|
211
|
-
return total;
|
|
212
|
-
}
|
|
213
|
-
/**
|
|
214
|
-
* Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
|
|
215
|
-
* the SSM state plus its convolution window, per block.
|
|
216
|
-
*/
|
|
217
|
-
function ssmStateBytes(facts, recurrentBlocks) {
|
|
218
|
-
if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
|
|
219
|
-
return 0;
|
|
220
|
-
const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
|
|
221
|
-
return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
|
|
222
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { EstimatorInput, EstimatorResult } from '../types.js';
|
|
2
|
-
export declare const WHISPER_ESTIMATOR_VERSION = "whisper-v1";
|
|
3
|
-
/**
|
|
4
|
-
* Estimates memory for a whisper.cpp transcription model.
|
|
5
|
-
*
|
|
6
|
-
* Whisper's encoder work is fixed by its 30 s window rather than by a growing
|
|
7
|
-
* cache, so the shape is simpler than the LLM case: resident weights plus a
|
|
8
|
-
* working peak that scales with the window the caller feeds it.
|
|
9
|
-
*/
|
|
10
|
-
export declare function estimateWhisper(input: EstimatorInput): EstimatorResult;
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
export const WHISPER_ESTIMATOR_VERSION = 'whisper-v1';
|
|
2
|
-
/** Whisper's own default window, and therefore the calibration anchor. */
|
|
3
|
-
const CALIBRATION_WINDOW_MS = 30_000;
|
|
4
|
-
/**
|
|
5
|
-
* Estimates memory for a whisper.cpp transcription model.
|
|
6
|
-
*
|
|
7
|
-
* Whisper's encoder work is fixed by its 30 s window rather than by a growing
|
|
8
|
-
* cache, so the shape is simpler than the LLM case: resident weights plus a
|
|
9
|
-
* working peak that scales with the window the caller feeds it.
|
|
10
|
-
*/
|
|
11
|
-
export function estimateWhisper(input) {
|
|
12
|
-
const { profile, workload, calibration, extraArtifactBytes } = input;
|
|
13
|
-
if (workload.kind !== 'audio') {
|
|
14
|
-
return {
|
|
15
|
-
kind: 'unknown',
|
|
16
|
-
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
17
|
-
reasons: [`workload kind '${workload.kind}' is not supported by ${WHISPER_ESTIMATOR_VERSION}`]
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
// A zero here is an unmeasured placeholder, not a measurement: the harness
|
|
21
|
-
// writes `{ lower: 0, upper: 0 }` until its whisper pass has run. Consuming
|
|
22
|
-
// it would return a confident estimate whose entire audio working memory is
|
|
23
|
-
// zero — the same failure shape as a mis-subtracted KV cache.
|
|
24
|
-
if (calibration.audioWindowBytes.upper <= 0) {
|
|
25
|
-
return {
|
|
26
|
-
kind: 'unknown',
|
|
27
|
-
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
28
|
-
reasons: [
|
|
29
|
-
'the audio window coefficient for this platform has not been measured, so an audio estimate cannot be defended'
|
|
30
|
-
]
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
if (workload.streaming && calibration.audioStreamingBytes.upper <= 0) {
|
|
34
|
-
return {
|
|
35
|
-
kind: 'unknown',
|
|
36
|
-
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
37
|
-
reasons: [
|
|
38
|
-
'the streaming-session coefficient for this platform has not been measured, so a streaming audio estimate cannot be defended'
|
|
39
|
-
]
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
const assumptions = [];
|
|
43
|
-
const reasons = [];
|
|
44
|
-
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
45
|
-
const persistent = {
|
|
46
|
-
lower: artifactBytes,
|
|
47
|
-
upper: Math.ceil(artifactBytes * calibration.weightUpperCoeff)
|
|
48
|
-
};
|
|
49
|
-
if (extraArtifactBytes > 0) {
|
|
50
|
-
assumptions.push('companion artifacts passed in `artifacts` (e.g. a VAD model) are counted at full size');
|
|
51
|
-
}
|
|
52
|
-
const windows = workload.windowMs / CALIBRATION_WINDOW_MS;
|
|
53
|
-
const streamingLower = workload.streaming ? calibration.audioStreamingBytes.lower : 0;
|
|
54
|
-
const streamingUpper = workload.streaming ? calibration.audioStreamingBytes.upper : 0;
|
|
55
|
-
const working = {
|
|
56
|
-
lower: calibration.fixedOverheadBytes.lower +
|
|
57
|
-
calibration.audioWindowBytes.lower * windows +
|
|
58
|
-
streamingLower,
|
|
59
|
-
upper: calibration.fixedOverheadBytes.upper +
|
|
60
|
-
calibration.audioWindowBytes.upper * windows +
|
|
61
|
-
streamingUpper
|
|
62
|
-
};
|
|
63
|
-
reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms`);
|
|
64
|
-
assumptions.push('the audio window scales working memory linearly from the engine’s 30 s default; longer windows are chunked by the engine rather than held whole');
|
|
65
|
-
if (workload.streaming) {
|
|
66
|
-
assumptions.push('a streaming session is assumed to keep one calibrated stream buffer resident');
|
|
67
|
-
}
|
|
68
|
-
return {
|
|
69
|
-
kind: 'estimate',
|
|
70
|
-
estimatorVersion: WHISPER_ESTIMATOR_VERSION,
|
|
71
|
-
persistent,
|
|
72
|
-
working: { lower: Math.ceil(working.lower), upper: Math.ceil(working.upper) },
|
|
73
|
-
reasons,
|
|
74
|
-
assumptions
|
|
75
|
-
};
|
|
76
|
-
}
|