@qvac/inference 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/NOTICE +107 -33
  2. package/README.md +8 -5
  3. package/dist/api/assess-model-fit.d.ts +9 -4
  4. package/dist/api/assess-model-fit.js +9 -4
  5. package/dist/api/bci-transcribe.d.ts +4 -1
  6. package/dist/api/index.d.ts +1 -0
  7. package/dist/api/index.js +1 -0
  8. package/dist/api/rpc-server.d.ts +19 -0
  9. package/dist/api/rpc-server.js +44 -0
  10. package/dist/api/text-to-speech.d.ts +9 -0
  11. package/dist/api/text-to-speech.js +9 -0
  12. package/dist/api/transcribe.d.ts +11 -7
  13. package/dist/api/transcribe.js +47 -15
  14. package/dist/api/video-request.d.ts +5 -5
  15. package/dist/dispatch.js +39 -7
  16. package/dist/errors/index.d.ts +7 -0
  17. package/dist/errors/index.js +15 -0
  18. package/dist/handlers/assess-model-fit.d.ts +18 -6
  19. package/dist/handlers/assess-model-fit.js +215 -35
  20. package/dist/handlers/completion-orchestrate.js +17 -0
  21. package/dist/handlers/get-loaded-model-info.js +4 -1
  22. package/dist/handlers/get-model-info.js +29 -1
  23. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  24. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  25. package/dist/handlers/load-model/hyperdrive.js +2 -2
  26. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  27. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  28. package/dist/handlers/load-model/registry.js +59 -0
  29. package/dist/handlers/rpc-server.d.ts +23 -0
  30. package/dist/handlers/rpc-server.js +141 -0
  31. package/dist/index.d.ts +2 -1
  32. package/dist/index.js +2 -1
  33. package/dist/models/registry/models.d.ts +7731 -5672
  34. package/dist/models/registry/models.js +4652 -2526
  35. package/dist/models/registry/resource-profiles.js +481 -94
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  40. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  44. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
  45. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  46. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  47. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  48. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  49. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  50. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  51. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  52. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  54. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  55. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  56. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  57. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  58. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  59. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  60. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  61. package/dist/plugins/ops/bci-transcribe.js +18 -13
  62. package/dist/plugins/ops/load-model.js +2 -2
  63. package/dist/plugins/ops/transcribe.d.ts +4 -2
  64. package/dist/plugins/ops/transcribe.js +39 -2
  65. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  66. package/dist/plugins/ops/translate-stats.js +64 -7
  67. package/dist/plugins/ops/translate.js +20 -4
  68. package/dist/profiling/operation-metrics.js +56 -0
  69. package/dist/registry.js +4 -0
  70. package/dist/resources/model-fit/assess.d.ts +46 -2
  71. package/dist/resources/model-fit/assess.js +67 -3
  72. package/dist/resources/model-fit/estimators/llm.js +2 -2
  73. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  74. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
  75. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
  76. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  77. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  78. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  79. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  80. package/dist/resources/model-fit/floor.js +2 -2
  81. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  82. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  83. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  84. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  85. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  86. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  87. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  88. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  89. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  90. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  91. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  92. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  93. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  94. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  95. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  96. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  97. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  98. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  99. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  100. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  101. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  102. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  103. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  104. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  105. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  106. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  107. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  108. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  109. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  110. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  111. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  112. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  113. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  114. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  115. package/dist/resources/model-fit/platform.d.ts +9 -0
  116. package/dist/resources/model-fit/platform.js +28 -0
  117. package/dist/rpc/device-map.d.ts +13 -0
  118. package/dist/rpc/device-map.js +22 -0
  119. package/dist/rpc/discovery.d.ts +12 -0
  120. package/dist/rpc/discovery.js +225 -0
  121. package/dist/rpc/ggml-provider.d.ts +4 -0
  122. package/dist/rpc/ggml-provider.js +25 -0
  123. package/dist/rpc/index.d.ts +2 -0
  124. package/dist/rpc/index.js +1 -0
  125. package/dist/rpc/instance.d.ts +6 -0
  126. package/dist/rpc/instance.js +21 -0
  127. package/dist/rpc/inventory.d.ts +5 -0
  128. package/dist/rpc/inventory.js +144 -0
  129. package/dist/rpc/manager.d.ts +14 -0
  130. package/dist/rpc/manager.js +174 -0
  131. package/dist/rpc/network.d.ts +11 -0
  132. package/dist/rpc/network.js +85 -0
  133. package/dist/rpc/provider.d.ts +6 -0
  134. package/dist/rpc/provider.js +24 -0
  135. package/dist/runtime/lifecycle.js +31 -13
  136. package/dist/runtime/request-context.d.ts +1 -1
  137. package/dist/runtime/state.js +5 -0
  138. package/dist/schemas/assess-model-fit.d.ts +284 -77
  139. package/dist/schemas/assess-model-fit.js +93 -55
  140. package/dist/schemas/audio-gen.d.ts +6 -6
  141. package/dist/schemas/audio-gen.js +2 -2
  142. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  143. package/dist/schemas/batch-completion-stream.js +5 -3
  144. package/dist/schemas/bci-config.js +1 -1
  145. package/dist/schemas/bci.d.ts +86 -1
  146. package/dist/schemas/bci.js +21 -2
  147. package/dist/schemas/cancel.d.ts +10 -0
  148. package/dist/schemas/cancel.js +2 -0
  149. package/dist/schemas/common.d.ts +765 -56
  150. package/dist/schemas/common.js +7 -0
  151. package/dist/schemas/completion-event.d.ts +6 -6
  152. package/dist/schemas/completion-stream.d.ts +27 -8
  153. package/dist/schemas/completion-stream.js +41 -4
  154. package/dist/schemas/config.d.ts +236 -30
  155. package/dist/schemas/config.js +10 -0
  156. package/dist/schemas/embed.d.ts +2 -2
  157. package/dist/schemas/error.js +13 -0
  158. package/dist/schemas/errors.d.ts +1 -0
  159. package/dist/schemas/errors.js +5 -0
  160. package/dist/schemas/index.d.ts +1 -0
  161. package/dist/schemas/index.js +1 -0
  162. package/dist/schemas/llamacpp-config.d.ts +105 -14
  163. package/dist/schemas/llamacpp-config.js +113 -3
  164. package/dist/schemas/load-model.d.ts +3102 -627
  165. package/dist/schemas/load-model.js +2 -2
  166. package/dist/schemas/mcp-adapter.d.ts +4 -0
  167. package/dist/schemas/model-config-schemas.d.ts +205 -5
  168. package/dist/schemas/model-info.d.ts +56 -0
  169. package/dist/schemas/model-info.js +5 -1
  170. package/dist/schemas/plugin.d.ts +8 -2
  171. package/dist/schemas/profiling.d.ts +4 -4
  172. package/dist/schemas/reload-config.d.ts +30 -0
  173. package/dist/schemas/rpc-server.d.ts +101 -0
  174. package/dist/schemas/rpc-server.js +87 -0
  175. package/dist/schemas/sdcpp-config.d.ts +7 -3
  176. package/dist/schemas/sdcpp-config.js +37 -2
  177. package/dist/schemas/system-resources.d.ts +4 -4
  178. package/dist/schemas/text-to-speech.d.ts +529 -1
  179. package/dist/schemas/text-to-speech.js +122 -12
  180. package/dist/schemas/tools.d.ts +2 -0
  181. package/dist/schemas/tools.js +8 -0
  182. package/dist/schemas/transcription-config.d.ts +5 -0
  183. package/dist/schemas/transcription-config.js +20 -2
  184. package/dist/schemas/transcription.d.ts +164 -0
  185. package/dist/schemas/transcription.js +71 -4
  186. package/dist/schemas/vla.d.ts +1 -1
  187. package/dist/surface.d.ts +4 -1
  188. package/dist/surface.js +3 -1
  189. package/dist/utils/addon-responses.d.ts +5 -1
  190. package/dist/utils/archive.js +5 -1
  191. package/dist/utils/asr-diagnostics.d.ts +12 -0
  192. package/dist/utils/asr-diagnostics.js +42 -0
  193. package/dist/utils/asr-events.d.ts +1 -0
  194. package/dist/utils/asr-events.js +3 -2
  195. package/dist/utils/cache/paths.js +3 -2
  196. package/dist/utils/mcp-adapter.js +7 -1
  197. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  198. package/dist/utils/multi-gpu-mobile.js +8 -6
  199. package/dist/utils/path-security.d.ts +5 -0
  200. package/dist/utils/path-security.js +14 -0
  201. package/dist/utils/tool-helpers.d.ts +4 -0
  202. package/dist/utils/tool-helpers.js +2 -0
  203. package/dist/utils/tools/defer.d.ts +93 -0
  204. package/dist/utils/tools/defer.js +290 -0
  205. package/dist/utils/tools/index.d.ts +1 -0
  206. package/dist/utils/tools/index.js +1 -0
  207. package/dist/utils/transcribe-metadata.d.ts +21 -7
  208. package/dist/utils/transcribe-metadata.js +24 -8
  209. package/dist/utils/tts-stats.d.ts +6 -0
  210. package/dist/utils/tts-stats.js +14 -1
  211. package/package.json +44 -30
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  213. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
@@ -1,171 +0,0 @@
1
- import { ModelType } from '../../../schemas/index.js';
2
- import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
3
- import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
4
- /**
5
- * Keys `@qvac/model-fit` reads as load evidence, in the spelling the SDK's own
6
- * completion/embedding transforms emit. The package canonicalizes `_` to `-`,
7
- * so `ctx_size` and `ctx-size` reach the same native setting.
8
- *
9
- * This mirrors `SUPPORTED_LOAD_KEYS` in the package's `LlamaLoadConfig.cpp`
10
- * intersected with what the two transforms can produce. It is a duplicated
11
- * policy for the duration of the experiment: a key added to a load config
12
- * without being classified here must not silently change the question the
13
- * fitter answers, so `partitionParams` refuses the request instead.
14
- */
15
- const FIT_LOAD_KEYS = {
16
- completion: [
17
- 'device',
18
- 'ctx_size',
19
- 'gpu_layers',
20
- 'load_mode',
21
- 'parallel',
22
- 'cache-type-k',
23
- 'cache-type-v',
24
- // Alters KV/compute memory; `model-fit` reads it as evidence.
25
- 'flash-attn',
26
- 'main-gpu',
27
- 'split-mode',
28
- 'tensor-split'
29
- ],
30
- embedding: [
31
- 'device',
32
- 'gpu_layers',
33
- 'batch_size',
34
- 'flash_attn',
35
- 'main-gpu',
36
- 'split-mode',
37
- 'tensor-split'
38
- ]
39
- };
40
- /**
41
- * Keys that reach the addon but cannot move a load's memory footprint —
42
- * sampling, generation, logging, and JS-side presentation settings. They are
43
- * dropped rather than forwarded: `@qvac/model-fit` rejects everything outside
44
- * its own allowlist, and a key it deliberately ignores must not turn a
45
- * supported load into `unsupported-config`.
46
- */
47
- const NON_FIT_KEYS = {
48
- completion: [
49
- 'temp',
50
- 'top_p',
51
- 'top_k',
52
- 'seed',
53
- 'predict',
54
- 'presence_penalty',
55
- 'frequency_penalty',
56
- 'repeat_penalty',
57
- 'reverse_prompt',
58
- 'n_discarded',
59
- 'tools',
60
- 'verbosity',
61
- 'reasoning_budget',
62
- 'image_tile_mode',
63
- 'image_no_upscale',
64
- 'mmproj-use-gpu',
65
- 'openclCacheDir'
66
- ],
67
- embedding: ['pooling', 'attention', 'embd_normalize', 'verbosity', 'openclCacheDir']
68
- };
69
- /**
70
- * Load-config keys that describe a shape the fitter cannot answer for.
71
- * `projection_model_src` reaches here when the config still carries the
72
- * unresolved source, ahead of the resolved-artifact refusal below.
73
- */
74
- const UNSUPPORTED_KEYS = ['lora', 'projection_model_src'];
75
- /**
76
- * A CPU load's weights stay file-backed and evictable, so the fitter projects
77
- * `fits` for nearly any model the OS could page through. Measured on a 24 GiB
78
- * M4 Pro against `@qvac/model-fit@0.8.0`: an 18.3 GiB model at 32k context
79
- * still reports `fits` on `device: 'cpu'` (and it does decode — at 0.1 tok/s).
80
- * That answer carries no admission information, so it is refused here rather
81
- * than spending a fit call to produce it.
82
- */
83
- function isCpuLoad(params) {
84
- return params['device']?.toLowerCase() === 'cpu';
85
- }
86
- function loadKindFor(modelType) {
87
- if (modelType === ModelType.llamacppCompletion)
88
- return 'completion';
89
- if (modelType === ModelType.llamacppEmbedding)
90
- return 'embedding';
91
- return undefined;
92
- }
93
- function unsupported(detail) {
94
- return { supported: false, detail };
95
- }
96
- function partitionParams(loadKind, transformed) {
97
- const forwarded = new Set(FIT_LOAD_KEYS[loadKind]);
98
- const dropped = new Set(NON_FIT_KEYS[loadKind]);
99
- const params = {};
100
- for (const [key, value] of Object.entries(transformed)) {
101
- if (UNSUPPORTED_KEYS.includes(key)) {
102
- return { detail: `unsupported load setting: ${key}` };
103
- }
104
- if (forwarded.has(key)) {
105
- params[key] = value;
106
- continue;
107
- }
108
- if (dropped.has(key))
109
- continue;
110
- // Neither fit evidence nor a known non-memory setting: this load carries a
111
- // setting the SDK cannot classify, so it must not be answered for.
112
- return { detail: `unclassified load setting: ${key}` };
113
- }
114
- return { params };
115
- }
116
- /**
117
- * Pins the requested context so the fitter reports on the exact load the SDK
118
- * is about to run. Left unset for an auto context (`0`, "use the model's
119
- * trained context"), where the package's own floor applies and the fitter
120
- * stays free to reduce.
121
- */
122
- function contextFloor(params) {
123
- const ctxSize = params['ctx_size'];
124
- if (ctxSize === undefined)
125
- return undefined;
126
- const parsed = Number(ctxSize);
127
- if (!Number.isSafeInteger(parsed) || parsed <= 0)
128
- return undefined;
129
- return parsed;
130
- }
131
- /**
132
- * Builds a protocol-v2 fit request from the same resolved model config the real
133
- * load is about to use, or explains why this load cannot be answered for.
134
- *
135
- * Structural shapes the SDK owns are refused here so the fitter is not asked
136
- * a question it cannot answer. Value-level policy (device names, symbolic GPU
137
- * selection, context bounds) stays inside `@qvac/model-fit`.
138
- */
139
- export function createLlamaFitRequest(params) {
140
- const loadKind = loadKindFor(params.modelType);
141
- if (loadKind === undefined) {
142
- return unsupported(`model type is not a llama.cpp load: ${params.modelType}`);
143
- }
144
- if (params.isShardedModel) {
145
- return unsupported('sharded models are not representable');
146
- }
147
- if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
148
- return unsupported('multimodal projection loads are not representable');
149
- }
150
- const modelConfig = (params.modelConfig ?? {});
151
- const transformed = loadKind === 'completion'
152
- ? transformLlmConfig(modelConfig)
153
- : transformEmbedConfig(modelConfig);
154
- const partitioned = partitionParams(loadKind, transformed);
155
- if ('detail' in partitioned)
156
- return unsupported(partitioned.detail);
157
- if (isCpuLoad(partitioned.params)) {
158
- return unsupported('cpu loads carry no device-memory evidence');
159
- }
160
- const nCtxMin = contextFloor(partitioned.params);
161
- return {
162
- supported: true,
163
- loadKind,
164
- config: {
165
- // Sharded loads are refused above, so the resolved path is the whole model.
166
- modelPath: params.modelPath,
167
- params: partitioned.params,
168
- ...(nCtxMin !== undefined && { nCtxMin })
169
- }
170
- };
171
- }