@qvac/inference 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/NOTICE +50 -25
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +4 -1
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.js +1 -1
  9. package/dist/api/decide.d.ts +26 -0
  10. package/dist/api/decide.js +26 -0
  11. package/dist/api/get-model-info.d.ts +1 -1
  12. package/dist/api/index.d.ts +1 -0
  13. package/dist/api/index.js +1 -0
  14. package/dist/api/text-to-speech.d.ts +1 -0
  15. package/dist/api/text-to-speech.js +15 -19
  16. package/dist/api/transcribe.js +5 -2
  17. package/dist/api/video-request.d.ts +5 -5
  18. package/dist/api/video.d.ts +5 -4
  19. package/dist/api/video.js +5 -4
  20. package/dist/errors/index.d.ts +3 -0
  21. package/dist/errors/index.js +5 -0
  22. package/dist/handlers/assess-model-fit.d.ts +5 -3
  23. package/dist/handlers/assess-model-fit.js +80 -28
  24. package/dist/handlers/get-loaded-model-info.js +1 -2
  25. package/dist/handlers/load-model/safe-fetch.js +5 -6
  26. package/dist/handlers/registry.d.ts +3 -1
  27. package/dist/handlers/registry.js +3 -3
  28. package/dist/index.d.ts +1 -1
  29. package/dist/index.js +1 -1
  30. package/dist/logging/logger.js +1 -1
  31. package/dist/logging/utils.js +1 -1
  32. package/dist/models/registry/models.d.ts +563 -816
  33. package/dist/models/registry/models.js +2873 -3110
  34. package/dist/models/registry/resource-profiles.d.ts +1 -2
  35. package/dist/models/registry/resource-profiles.js +203 -2
  36. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  37. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  38. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  39. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  40. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
  41. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  42. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  43. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
  44. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
  45. package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
  46. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
  47. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
  48. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
  49. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  50. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  51. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
  52. package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
  53. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
  54. package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
  55. package/dist/plugins/builtin/index.d.ts +2 -0
  56. package/dist/plugins/builtin/index.js +2 -0
  57. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
  58. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
  59. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
  60. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
  61. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  62. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  63. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
  64. package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
  65. package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
  66. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
  67. package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
  68. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  69. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  70. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
  71. package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
  72. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
  73. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
  74. package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
  75. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  76. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  77. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  78. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  79. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
  80. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
  81. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
  82. package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
  83. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
  84. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
  85. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
  86. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
  87. package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
  88. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  89. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  90. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
  91. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
  92. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
  93. package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
  94. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
  95. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
  96. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  97. package/dist/plugins/ops/kv-cache-utils.js +10 -25
  98. package/dist/plugins/ops/load-model.js +21 -16
  99. package/dist/plugins/ops/transcribe.js +17 -1
  100. package/dist/registry.js +1 -0
  101. package/dist/resources/model-fit/assess.d.ts +15 -23
  102. package/dist/resources/model-fit/assess.js +207 -271
  103. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
  104. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
  105. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
  106. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
  107. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
  108. package/dist/resources/model-fit/floor.d.ts +10 -10
  109. package/dist/resources/model-fit/floor.js +13 -13
  110. package/dist/resources/model-fit/kv-cache.d.ts +23 -0
  111. package/dist/resources/model-fit/kv-cache.js +85 -0
  112. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
  113. package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
  114. package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
  115. package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
  116. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
  117. package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
  118. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
  119. package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
  120. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
  121. package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
  122. package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
  123. package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
  124. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
  125. package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
  126. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
  127. package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
  128. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
  129. package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
  130. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
  131. package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
  132. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
  133. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
  134. package/dist/resources/model-fit/platform.d.ts +4 -4
  135. package/dist/resources/model-fit/platform.js +6 -8
  136. package/dist/resources/model-fit/types.d.ts +3 -117
  137. package/dist/rpc/inventory.js +5 -7
  138. package/dist/runtime/env.d.ts +0 -2
  139. package/dist/runtime/env.js +2 -9
  140. package/dist/runtime/model-config-utils.js +6 -1
  141. package/dist/runtime/request-context.d.ts +1 -1
  142. package/dist/runtime/request-context.js +8 -0
  143. package/dist/runtime/state.js +5 -0
  144. package/dist/schemas/assess-model-fit.d.ts +100 -43
  145. package/dist/schemas/assess-model-fit.js +66 -29
  146. package/dist/schemas/audio-gen.d.ts +186 -18
  147. package/dist/schemas/audio-gen.js +180 -48
  148. package/dist/schemas/batch-completion-stream.d.ts +202 -34
  149. package/dist/schemas/bci-config.d.ts +2 -0
  150. package/dist/schemas/bci.d.ts +23 -2
  151. package/dist/schemas/cancel.d.ts +5 -0
  152. package/dist/schemas/cancel.js +1 -0
  153. package/dist/schemas/common.d.ts +1083 -158
  154. package/dist/schemas/common.js +5 -2
  155. package/dist/schemas/completion-stream.d.ts +232 -37
  156. package/dist/schemas/completion-stream.js +64 -20
  157. package/dist/schemas/config.d.ts +477 -12
  158. package/dist/schemas/config.js +21 -1
  159. package/dist/schemas/decision.d.ts +602 -0
  160. package/dist/schemas/decision.js +205 -0
  161. package/dist/schemas/download-asset.d.ts +7 -3
  162. package/dist/schemas/engine-addon-map.d.ts +7 -0
  163. package/dist/schemas/engine-addon-map.js +23 -3
  164. package/dist/schemas/errors.d.ts +1 -0
  165. package/dist/schemas/errors.js +5 -0
  166. package/dist/schemas/index.d.ts +2 -0
  167. package/dist/schemas/index.js +2 -0
  168. package/dist/schemas/llamacpp-config.d.ts +42 -2
  169. package/dist/schemas/llamacpp-config.js +27 -0
  170. package/dist/schemas/load-model.d.ts +5247 -1309
  171. package/dist/schemas/load-model.js +88 -9
  172. package/dist/schemas/model-config-schemas.d.ts +233 -8
  173. package/dist/schemas/model-config-schemas.js +4 -1
  174. package/dist/schemas/model-fit-policy.d.ts +17 -0
  175. package/dist/schemas/model-fit-policy.js +17 -0
  176. package/dist/schemas/model-info.d.ts +20 -2
  177. package/dist/schemas/model-info.js +2 -1
  178. package/dist/schemas/model-resource-profile.d.ts +3 -4
  179. package/dist/schemas/model-resource-profile.js +1 -4
  180. package/dist/schemas/model-src-utils.d.ts +10 -2
  181. package/dist/schemas/model-types.d.ts +10 -1
  182. package/dist/schemas/model-types.js +3 -0
  183. package/dist/schemas/ocr.d.ts +74 -0
  184. package/dist/schemas/ocr.js +42 -0
  185. package/dist/schemas/plugin.d.ts +8 -2
  186. package/dist/schemas/plugin.js +6 -0
  187. package/dist/schemas/profiling.d.ts +4 -4
  188. package/dist/schemas/registry.d.ts +26 -0
  189. package/dist/schemas/registry.js +3 -0
  190. package/dist/schemas/reload-config.d.ts +10 -2
  191. package/dist/schemas/rpc-server.js +3 -3
  192. package/dist/schemas/sdcpp-config.d.ts +26 -0
  193. package/dist/schemas/sdcpp-config.js +8 -7
  194. package/dist/schemas/system-resources.d.ts +4 -4
  195. package/dist/schemas/text-to-speech.d.ts +402 -3
  196. package/dist/schemas/text-to-speech.js +129 -11
  197. package/dist/schemas/tools.d.ts +13 -4
  198. package/dist/schemas/tools.js +18 -6
  199. package/dist/schemas/transcription-config.d.ts +15 -0
  200. package/dist/schemas/transcription-config.js +38 -3
  201. package/dist/schemas/transcription.d.ts +46 -4
  202. package/dist/schemas/transcription.js +51 -4
  203. package/dist/schemas/translation-config.d.ts +30 -10
  204. package/dist/schemas/vla.d.ts +9 -2
  205. package/dist/schemas/vla.js +9 -5
  206. package/dist/surface.d.ts +1 -1
  207. package/dist/surface.js +1 -1
  208. package/dist/utils/addon-responses.d.ts +1 -0
  209. package/dist/utils/asr-events.d.ts +1 -0
  210. package/dist/utils/asr-events.js +2 -1
  211. package/dist/utils/cache/types.d.ts +7 -0
  212. package/dist/utils/mcp-adapter.js +12 -8
  213. package/dist/utils/tool-helpers.d.ts +1 -1
  214. package/dist/utils/tool-helpers.js +20 -52
  215. package/dist/utils/tools/parsers/dsml.js +8 -5
  216. package/dist/utils/tools/parsers/qwen35.js +7 -4
  217. package/dist/utils/tools/shared.d.ts +3 -0
  218. package/dist/utils/tools/shared.js +62 -0
  219. package/dist/utils/transcribe-metadata.d.ts +8 -1
  220. package/dist/utils/transcribe-metadata.js +13 -3
  221. package/dist/utils/tts-stats.d.ts +1 -0
  222. package/package.json +41 -32
  223. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  224. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  225. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  226. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  227. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  228. package/dist/resources/model-fit/calibration/fit.js +0 -152
  229. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  230. package/dist/resources/model-fit/calibration/index.js +0 -75
  231. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  232. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  233. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  234. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  235. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  236. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  237. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  238. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  239. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  240. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  241. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  242. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  243. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  244. package/dist/resources/model-fit/estimators/llm.js +0 -222
  245. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  246. package/dist/resources/model-fit/estimators/whisper.js +0 -76
@@ -1,39 +0,0 @@
1
- /**
2
- * linux-x64-vulkan coefficients.
3
- *
4
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
5
- * to this file for how the numbers are derived and validated.
6
- */
7
- export const LINUX_X64_VULKAN_CALIBRATION = {
8
- weightUpperCoeff: 1.019,
9
- workingPeakBytes: {
10
- lower: 0,
11
- upper: 0
12
- },
13
- fixedOverheadBytes: {
14
- lower: 234775431,
15
- upper: 368894651
16
- },
17
- computeBufferBytesPerToken: {
18
- lower: 0,
19
- upper: 0
20
- },
21
- audioWindowBytes: {
22
- lower: 0,
23
- upper: 0
24
- },
25
- audioStreamingBytes: {
26
- lower: 0,
27
- upper: 0
28
- },
29
- validated: true,
30
- notes: [
31
- 'workingPeakBytes is an unsampled zero, not a measurement: this run polled no device counter during completions. Re-measure before trusting a per-completion budget on this backend'
32
- ],
33
- measuredAt: '2026-09-03',
34
- measuredOn: {
35
- backend: 'vulkan',
36
- device: 'NVIDIA RTX 4000 SFF Ada Generation',
37
- kvElementBytes: 1.0625
38
- }
39
- };
@@ -1,8 +0,0 @@
1
- import type { PlatformCalibration } from '../types.js';
2
- /**
3
- * linux-x64 coefficients.
4
- *
5
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
6
- * to this file for how the numbers are derived and validated.
7
- */
8
- export declare const LINUX_X64_CALIBRATION: PlatformCalibration;
@@ -1,38 +0,0 @@
1
- /**
2
- * linux-x64 coefficients.
3
- *
4
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
5
- * to this file for how the numbers are derived and validated.
6
- */
7
- export const LINUX_X64_CALIBRATION = {
8
- weightUpperCoeff: 1.01,
9
- workingPeakBytes: {
10
- lower: 0,
11
- upper: 8223130
12
- },
13
- fixedOverheadBytes: {
14
- lower: 0,
15
- upper: 19194251
16
- },
17
- computeBufferBytesPerToken: {
18
- lower: 927,
19
- upper: 1391
20
- },
21
- audioWindowBytes: {
22
- lower: 0,
23
- upper: 0
24
- },
25
- audioStreamingBytes: {
26
- lower: 0,
27
- upper: 0
28
- },
29
- validated: true,
30
- notes: [
31
- "weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
32
- ],
33
- measuredAt: '2026-09-03',
34
- measuredOn: {
35
- backend: 'cpu',
36
- kvElementBytes: 2
37
- }
38
- };
@@ -1,8 +0,0 @@
1
- import type { PlatformCalibration } from '../types.js';
2
- /**
3
- * win32-x64-vulkan-shared coefficients.
4
- *
5
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
6
- * to this file for how the numbers are derived and validated.
7
- */
8
- export declare const WIN32_X64_VULKAN_SHARED_CALIBRATION: PlatformCalibration;
@@ -1,39 +0,0 @@
1
- /**
2
- * win32-x64-vulkan-shared coefficients.
3
- *
4
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
5
- * to this file for how the numbers are derived and validated.
6
- */
7
- export const WIN32_X64_VULKAN_SHARED_CALIBRATION = {
8
- weightUpperCoeff: 2.044,
9
- workingPeakBytes: {
10
- lower: 0,
11
- upper: 13044941
12
- },
13
- fixedOverheadBytes: {
14
- lower: 239502621,
15
- upper: 421678470
16
- },
17
- computeBufferBytesPerToken: {
18
- lower: 5932,
19
- upper: 8898
20
- },
21
- audioWindowBytes: {
22
- lower: 0,
23
- upper: 0
24
- },
25
- audioStreamingBytes: {
26
- lower: 0,
27
- upper: 0
28
- },
29
- validated: true,
30
- notes: [
31
- "measured with 'main-gpu: integrated' and the SDK's default load mode, against process RSS: an integrated GPU allocates out of system RAM, so these coefficients belong to the system-memory basis and not to a device budget"
32
- ],
33
- measuredAt: '2026-09-03',
34
- measuredOn: {
35
- backend: 'vulkan',
36
- device: 'Intel(R) UHD Graphics 770',
37
- kvElementBytes: 1.0625
38
- }
39
- };
@@ -1,8 +0,0 @@
1
- import type { PlatformCalibration } from '../types.js';
2
- /**
3
- * win32-x64-vulkan coefficients.
4
- *
5
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
6
- * to this file for how the numbers are derived and validated.
7
- */
8
- export declare const WIN32_X64_VULKAN_CALIBRATION: PlatformCalibration;
@@ -1,36 +0,0 @@
1
- /**
2
- * win32-x64-vulkan coefficients.
3
- *
4
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
5
- * to this file for how the numbers are derived and validated.
6
- */
7
- export const WIN32_X64_VULKAN_CALIBRATION = {
8
- weightUpperCoeff: 1.018,
9
- workingPeakBytes: {
10
- lower: 0,
11
- upper: 6606029
12
- },
13
- fixedOverheadBytes: {
14
- lower: 231096560,
15
- upper: 364959355
16
- },
17
- computeBufferBytesPerToken: {
18
- lower: 0,
19
- upper: 0
20
- },
21
- audioWindowBytes: {
22
- lower: 0,
23
- upper: 0
24
- },
25
- audioStreamingBytes: {
26
- lower: 0,
27
- upper: 0
28
- },
29
- validated: true,
30
- measuredAt: '2026-09-04',
31
- measuredOn: {
32
- backend: 'vulkan',
33
- device: 'NVIDIA RTX 4000 SFF Ada Generation',
34
- kvElementBytes: 1.0625
35
- }
36
- };
@@ -1,8 +0,0 @@
1
- import type { PlatformCalibration } from '../types.js';
2
- /**
3
- * win32-x64 coefficients.
4
- *
5
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
6
- * to this file for how the numbers are derived and validated.
7
- */
8
- export declare const WIN32_X64_CALIBRATION: PlatformCalibration;
@@ -1,38 +0,0 @@
1
- /**
2
- * win32-x64 coefficients.
3
- *
4
- * Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
5
- * to this file for how the numbers are derived and validated.
6
- */
7
- export const WIN32_X64_CALIBRATION = {
8
- weightUpperCoeff: 1.01,
9
- workingPeakBytes: {
10
- lower: 0,
11
- upper: 10587341
12
- },
13
- fixedOverheadBytes: {
14
- lower: 55193195,
15
- upper: 134358525
16
- },
17
- computeBufferBytesPerToken: {
18
- lower: 0,
19
- upper: 0
20
- },
21
- audioWindowBytes: {
22
- lower: 0,
23
- upper: 0
24
- },
25
- audioStreamingBytes: {
26
- lower: 0,
27
- upper: 0
28
- },
29
- validated: true,
30
- notes: [
31
- "weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
32
- ],
33
- measuredAt: '2026-09-03',
34
- measuredOn: {
35
- backend: 'cpu',
36
- kvElementBytes: 2
37
- }
38
- };
@@ -1,46 +0,0 @@
1
- import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
2
- import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
3
- export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
4
- export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
5
- /**
6
- * Estimates memory for a llama.cpp completion or embedding model from catalog
7
- * metadata alone.
8
- *
9
- * Weights come from the artifact size; the KV cache is computed from the
10
- * transformer shape and the requested context. The estimate is a range, not a
11
- * number, because two things are genuinely undetermined before load: the
12
- * backend the engine will pick (which sets the default KV-cache type) and, for
13
- * some architectures, which blocks hold a full cache.
14
- */
15
- export declare function estimateLlm(input: EstimatorInput): EstimatorResult;
16
- /** A KV-cache element width range, with the reason it is that range. */
17
- export interface KvElementWidth {
18
- bytes: ByteRange;
19
- assumption: string;
20
- }
21
- /**
22
- * Picks the KV-cache element width range for this model on this device.
23
- *
24
- * Exported because the calibration harness has to subtract the cache the engine
25
- * *actually* allocated before it can fit the remaining overhead. Hard-coding a
26
- * width there would skew the fit by roughly 2× on a GPU backend — and because
27
- * the error scales with context, it would corrupt the per-token slope rather
28
- * than shifting the intercept. Sharing this rule keeps the two in step.
29
- *
30
- * @returns Lower/upper bytes per cache element, and the assumption that choice
31
- * rests on. Side-effect free so the harness can call it without an
32
- * assumptions array.
33
- */
34
- export declare function kvElementBytes(facts: GgufFacts, hasGpu: boolean): KvElementWidth;
35
- /**
36
- * KV-cache bytes at one fixed element width, clamped to the trained context.
37
- *
38
- * Exported for the calibration harness, which must subtract the cache the
39
- * engine actually allocated using the exact accounting the estimator uses —
40
- * a hand-rolled copy drifted once already (no sliding-window branch, `ceil`
41
- * where the estimator bounds with `floor`, no SSM state). A non-degenerate
42
- * range means part of the layout is engine-owned, so the allocation cannot be
43
- * known from the file alone and the model is unsuitable for calibration; the
44
- * harness aborts on it rather than guessing.
45
- */
46
- export declare function kvCacheBytesForWidth(facts: GgufFacts, contextTokens: number, bytesPerElement: number): ByteRange;
@@ -1,222 +0,0 @@
1
- export const LLM_ESTIMATOR_VERSION = 'llm-v1';
2
- /**
3
- * KV-cache element widths, in bytes per element.
4
- *
5
- * `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
6
- * on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
7
- * instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318),
8
- * which is why the bound is a range rather than a number. `q8_0` packs 32
9
- * elements into a 34-byte block.
10
- */
11
- const F16_BYTES_PER_ELEMENT = 2;
12
- const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
13
- /** Recurrent/SSM state is kept in f32. */
14
- const SSM_STATE_BYTES_PER_ELEMENT = 4;
15
- /**
16
- * Architectures that disable flash attention, so the GPU `q8_0` KV default
17
- * never applies and the cache stays `f16` on every backend.
18
- */
19
- function disablesFlashAttention(architecture) {
20
- return architecture.startsWith('bitnet');
21
- }
22
- // Shared with the computed floor so a mixed set carries one weights assumption, not two.
23
- export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
24
- /**
25
- * Estimates memory for a llama.cpp completion or embedding model from catalog
26
- * metadata alone.
27
- *
28
- * Weights come from the artifact size; the KV cache is computed from the
29
- * transformer shape and the requested context. The estimate is a range, not a
30
- * number, because two things are genuinely undetermined before load: the
31
- * backend the engine will pick (which sets the default KV-cache type) and, for
32
- * some architectures, which blocks hold a full cache.
33
- */
34
- export function estimateLlm(input) {
35
- const { profile, workload, calibration, extraArtifactBytes, hasGpu } = input;
36
- if (workload.kind !== 'llm') {
37
- return {
38
- kind: 'unknown',
39
- estimatorVersion: LLM_ESTIMATOR_VERSION,
40
- reasons: [`workload kind '${workload.kind}' is not supported by ${LLM_ESTIMATOR_VERSION}`]
41
- };
42
- }
43
- const facts = profile.ggufFacts;
44
- if (!facts) {
45
- return {
46
- kind: 'unknown',
47
- estimatorVersion: LLM_ESTIMATOR_VERSION,
48
- reasons: ['no GGUF metadata for this model in the catalog, so the KV cache cannot be sized']
49
- };
50
- }
51
- const assumptions = [];
52
- const reasons = [];
53
- // Weights: mapped at file size by default, so artifact bytes are the floor.
54
- // The upper coefficient covers the allocator's copy-on-write and alignment
55
- // slack measured during calibration.
56
- const artifactBytes = profile.artifactBytes + extraArtifactBytes;
57
- assumptions.push(LLAMA_WEIGHTS_ASSUMPTION);
58
- if (extraArtifactBytes > 0) {
59
- assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
60
- }
61
- // Context: never more than the model was trained for — llama.cpp clamps.
62
- let contextTokens = workload.contextTokens ?? facts.contextLength;
63
- if (contextTokens > facts.contextLength) {
64
- assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
65
- contextTokens = facts.contextLength;
66
- }
67
- const element = kvElementBytes(facts, hasGpu);
68
- assumptions.push(element.assumption);
69
- const kv = kvCacheBytes(facts, contextTokens, element.bytes, assumptions, reasons);
70
- // llama.cpp builds the context at load — KV cache, engine overhead, compute
71
- // buffers — so all of it is resident for the model's lifetime. Parking any of
72
- // it in `working` would let `sequential` count only the largest. What a
73
- // completion adds on top is released afterwards, so that part is `working`.
74
- const persistent = {
75
- lower: Math.ceil(artifactBytes +
76
- kv.lower +
77
- calibration.fixedOverheadBytes.lower +
78
- calibration.computeBufferBytesPerToken.lower * contextTokens),
79
- upper: Math.ceil(artifactBytes * calibration.weightUpperCoeff +
80
- kv.upper +
81
- calibration.fixedOverheadBytes.upper +
82
- calibration.computeBufferBytesPerToken.upper * contextTokens)
83
- };
84
- assumptions.push('the KV cache, engine overhead and compute buffers count as resident for the model’s whole lifetime; llama.cpp allocates them when the model loads, not per operation');
85
- if (calibration.workingPeakBytes && calibration.workingPeakBytes.upper > 0) {
86
- assumptions.push('one operation is assumed in flight per model; the working peak is what a single completion was measured to add on top of the resident cost');
87
- }
88
- assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
89
- return {
90
- kind: 'estimate',
91
- estimatorVersion: LLM_ESTIMATOR_VERSION,
92
- persistent,
93
- working: {
94
- lower: calibration.workingPeakBytes?.lower ?? 0,
95
- upper: calibration.workingPeakBytes?.upper ?? 0
96
- },
97
- reasons,
98
- assumptions
99
- };
100
- }
101
- /**
102
- * Picks the KV-cache element width range for this model on this device.
103
- *
104
- * Exported because the calibration harness has to subtract the cache the engine
105
- * *actually* allocated before it can fit the remaining overhead. Hard-coding a
106
- * width there would skew the fit by roughly 2× on a GPU backend — and because
107
- * the error scales with context, it would corrupt the per-token slope rather
108
- * than shifting the intercept. Sharing this rule keeps the two in step.
109
- *
110
- * @returns Lower/upper bytes per cache element, and the assumption that choice
111
- * rests on. Side-effect free so the harness can call it without an
112
- * assumptions array.
113
- */
114
- export function kvElementBytes(facts, hasGpu) {
115
- if (disablesFlashAttention(facts.architecture)) {
116
- return {
117
- bytes: { lower: F16_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
118
- assumption: `${facts.architecture} loads with flash attention off, so the KV cache stays f16 on every backend`
119
- };
120
- }
121
- if (hasGpu) {
122
- return {
123
- bytes: { lower: Q8_0_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
124
- assumption: 'a GPU is present, so the engine may default the KV cache to q8_0 (lower bound) or keep f16 on a CPU or OpenCL backend (upper bound)'
125
- };
126
- }
127
- return {
128
- bytes: { lower: F16_BYTES_PER_ELEMENT, upper: F16_BYTES_PER_ELEMENT },
129
- assumption: 'no GPU reported, so the CPU f16 KV-cache default applies'
130
- };
131
- }
132
- /**
133
- * KV-cache bytes at one fixed element width, clamped to the trained context.
134
- *
135
- * Exported for the calibration harness, which must subtract the cache the
136
- * engine actually allocated using the exact accounting the estimator uses —
137
- * a hand-rolled copy drifted once already (no sliding-window branch, `ceil`
138
- * where the estimator bounds with `floor`, no SSM state). A non-degenerate
139
- * range means part of the layout is engine-owned, so the allocation cannot be
140
- * known from the file alone and the model is unsuitable for calibration; the
141
- * harness aborts on it rather than guessing.
142
- */
143
- export function kvCacheBytesForWidth(facts, contextTokens, bytesPerElement) {
144
- const tokens = Math.min(contextTokens, facts.contextLength);
145
- return kvCacheBytes(facts, tokens, { lower: bytesPerElement, upper: bytesPerElement }, [], []);
146
- }
147
- /**
148
- * Sizes the KV cache for the requested context.
149
- *
150
- * Three cases, in order of how much the file actually tells us:
151
- *
152
- * 1. **Per-layer classes** — the file describes attention per block, so the
153
- * cache is summed exactly, with sliding-window blocks capped at their window.
154
- * 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
155
- * blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
156
- * blocks are which is engine-owned, so the count is bounded, not fixed.
157
- * 3. **Flat** — every block holds the same cache. When the file declares a
158
- * sliding window but no per-layer pattern, the pattern lives in the engine:
159
- * the bound then spans "every block windowed" to "every block full", which is
160
- * wide on purpose.
161
- */
162
- function kvCacheBytes(facts, contextTokens, elementBytes, assumptions, reasons) {
163
- if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
164
- reasons.push('KV cache summed per layer class from the file’s per-block attention metadata');
165
- return {
166
- lower: layerClassBytes(facts.kvLayerClasses, facts, contextTokens, elementBytes.lower),
167
- upper: layerClassBytes(facts.kvLayerClasses, facts, contextTokens, elementBytes.upper)
168
- };
169
- }
170
- const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
171
- if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
172
- const interval = facts.fullAttentionInterval;
173
- const fullBlocksLower = Math.floor(facts.blockCount / interval);
174
- const fullBlocksUpper = Math.ceil(facts.blockCount / interval);
175
- const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocksLower);
176
- assumptions.push(`${facts.architecture} keeps full attention every ${interval} blocks; the remaining blocks hold a fixed-size recurrent state instead of a KV cache, and which blocks are which is engine-owned`);
177
- reasons.push(`KV cache sized for ${fullBlocksLower}–${fullBlocksUpper} of ${facts.blockCount} blocks (hybrid attention)`);
178
- return {
179
- lower: fullBlocksLower * perBlockPerToken * contextTokens * elementBytes.lower + ssm,
180
- upper: fullBlocksUpper * perBlockPerToken * contextTokens * elementBytes.upper + ssm
181
- };
182
- }
183
- if (facts.slidingWindow) {
184
- const windowedTokens = Math.min(contextTokens, facts.slidingWindow);
185
- assumptions.push(`${facts.architecture} uses sliding-window attention with a ${facts.slidingWindow}-token window, but the file does not say which blocks are windowed; the bound spans every block windowed to every block full`);
186
- reasons.push('sliding-window layer pattern is engine-owned, so the KV bound is wide');
187
- return {
188
- lower: facts.blockCount * perBlockPerToken * windowedTokens * elementBytes.lower,
189
- upper: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.upper
190
- };
191
- }
192
- reasons.push(`KV cache sized for all ${facts.blockCount} blocks at ${contextTokens} tokens`);
193
- return {
194
- lower: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.lower,
195
- upper: facts.blockCount * perBlockPerToken * contextTokens * elementBytes.upper
196
- };
197
- }
198
- function layerClassBytes(classes, facts, contextTokens, elementBytes) {
199
- let total = 0;
200
- for (const layerClass of classes) {
201
- const tokens = layerClass.windowed && facts.slidingWindow
202
- ? Math.min(contextTokens, facts.slidingWindow)
203
- : contextTokens;
204
- total +=
205
- layerClass.count *
206
- layerClass.headCountKv *
207
- (layerClass.keyLength + layerClass.valueLength) *
208
- tokens *
209
- elementBytes;
210
- }
211
- return total;
212
- }
213
- /**
214
- * Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
215
- * the SSM state plus its convolution window, per block.
216
- */
217
- function ssmStateBytes(facts, recurrentBlocks) {
218
- if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
219
- return 0;
220
- const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
221
- return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
222
- }
@@ -1,10 +0,0 @@
1
- import type { EstimatorInput, EstimatorResult } from '../types.js';
2
- export declare const WHISPER_ESTIMATOR_VERSION = "whisper-v1";
3
- /**
4
- * Estimates memory for a whisper.cpp transcription model.
5
- *
6
- * Whisper's encoder work is fixed by its 30 s window rather than by a growing
7
- * cache, so the shape is simpler than the LLM case: resident weights plus a
8
- * working peak that scales with the window the caller feeds it.
9
- */
10
- export declare function estimateWhisper(input: EstimatorInput): EstimatorResult;
@@ -1,76 +0,0 @@
1
- export const WHISPER_ESTIMATOR_VERSION = 'whisper-v1';
2
- /** Whisper's own default window, and therefore the calibration anchor. */
3
- const CALIBRATION_WINDOW_MS = 30_000;
4
- /**
5
- * Estimates memory for a whisper.cpp transcription model.
6
- *
7
- * Whisper's encoder work is fixed by its 30 s window rather than by a growing
8
- * cache, so the shape is simpler than the LLM case: resident weights plus a
9
- * working peak that scales with the window the caller feeds it.
10
- */
11
- export function estimateWhisper(input) {
12
- const { profile, workload, calibration, extraArtifactBytes } = input;
13
- if (workload.kind !== 'audio') {
14
- return {
15
- kind: 'unknown',
16
- estimatorVersion: WHISPER_ESTIMATOR_VERSION,
17
- reasons: [`workload kind '${workload.kind}' is not supported by ${WHISPER_ESTIMATOR_VERSION}`]
18
- };
19
- }
20
- // A zero here is an unmeasured placeholder, not a measurement: the harness
21
- // writes `{ lower: 0, upper: 0 }` until its whisper pass has run. Consuming
22
- // it would return a confident estimate whose entire audio working memory is
23
- // zero — the same failure shape as a mis-subtracted KV cache.
24
- if (calibration.audioWindowBytes.upper <= 0) {
25
- return {
26
- kind: 'unknown',
27
- estimatorVersion: WHISPER_ESTIMATOR_VERSION,
28
- reasons: [
29
- 'the audio window coefficient for this platform has not been measured, so an audio estimate cannot be defended'
30
- ]
31
- };
32
- }
33
- if (workload.streaming && calibration.audioStreamingBytes.upper <= 0) {
34
- return {
35
- kind: 'unknown',
36
- estimatorVersion: WHISPER_ESTIMATOR_VERSION,
37
- reasons: [
38
- 'the streaming-session coefficient for this platform has not been measured, so a streaming audio estimate cannot be defended'
39
- ]
40
- };
41
- }
42
- const assumptions = [];
43
- const reasons = [];
44
- const artifactBytes = profile.artifactBytes + extraArtifactBytes;
45
- const persistent = {
46
- lower: artifactBytes,
47
- upper: Math.ceil(artifactBytes * calibration.weightUpperCoeff)
48
- };
49
- if (extraArtifactBytes > 0) {
50
- assumptions.push('companion artifacts passed in `artifacts` (e.g. a VAD model) are counted at full size');
51
- }
52
- const windows = workload.windowMs / CALIBRATION_WINDOW_MS;
53
- const streamingLower = workload.streaming ? calibration.audioStreamingBytes.lower : 0;
54
- const streamingUpper = workload.streaming ? calibration.audioStreamingBytes.upper : 0;
55
- const working = {
56
- lower: calibration.fixedOverheadBytes.lower +
57
- calibration.audioWindowBytes.lower * windows +
58
- streamingLower,
59
- upper: calibration.fixedOverheadBytes.upper +
60
- calibration.audioWindowBytes.upper * windows +
61
- streamingUpper
62
- };
63
- reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms`);
64
- assumptions.push('the audio window scales working memory linearly from the engine’s 30 s default; longer windows are chunked by the engine rather than held whole');
65
- if (workload.streaming) {
66
- assumptions.push('a streaming session is assumed to keep one calibrated stream buffer resident');
67
- }
68
- return {
69
- kind: 'estimate',
70
- estimatorVersion: WHISPER_ESTIMATOR_VERSION,
71
- persistent,
72
- working: { lower: Math.ceil(working.lower), upper: Math.ceil(working.upper) },
73
- reasons,
74
- assumptions
75
- };
76
- }