@qvac/inference 0.20.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/NOTICE +90 -24
  2. package/README.md +8 -5
  3. package/dist/api/bci-transcribe.d.ts +4 -1
  4. package/dist/api/index.d.ts +1 -0
  5. package/dist/api/index.js +1 -0
  6. package/dist/api/rpc-server.d.ts +19 -0
  7. package/dist/api/rpc-server.js +44 -0
  8. package/dist/api/text-to-speech.d.ts +9 -0
  9. package/dist/api/text-to-speech.js +9 -0
  10. package/dist/api/transcribe.d.ts +11 -7
  11. package/dist/api/transcribe.js +47 -15
  12. package/dist/api/video-request.d.ts +5 -5
  13. package/dist/dispatch.js +39 -7
  14. package/dist/errors/index.d.ts +7 -0
  15. package/dist/errors/index.js +15 -0
  16. package/dist/handlers/assess-model-fit.d.ts +8 -20
  17. package/dist/handlers/assess-model-fit.js +195 -77
  18. package/dist/handlers/completion-orchestrate.js +17 -0
  19. package/dist/handlers/get-loaded-model-info.js +4 -1
  20. package/dist/handlers/get-model-info.js +29 -1
  21. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  22. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  23. package/dist/handlers/load-model/hyperdrive.js +2 -2
  24. package/dist/handlers/load-model/registry.js +59 -0
  25. package/dist/handlers/rpc-server.d.ts +23 -0
  26. package/dist/handlers/rpc-server.js +141 -0
  27. package/dist/index.d.ts +2 -1
  28. package/dist/index.js +2 -1
  29. package/dist/models/registry/models.d.ts +7731 -5672
  30. package/dist/models/registry/models.js +4652 -2526
  31. package/dist/models/registry/resource-profiles.js +481 -94
  32. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  34. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  36. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  40. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
  41. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  42. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  43. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  44. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  45. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  46. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  47. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  48. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  49. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  50. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  51. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  52. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  53. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  54. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  55. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  56. package/dist/plugins/ops/bci-transcribe.js +18 -13
  57. package/dist/plugins/ops/load-model.js +2 -2
  58. package/dist/plugins/ops/transcribe.d.ts +4 -2
  59. package/dist/plugins/ops/transcribe.js +39 -2
  60. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  61. package/dist/plugins/ops/translate-stats.js +64 -7
  62. package/dist/plugins/ops/translate.js +20 -4
  63. package/dist/profiling/operation-metrics.js +56 -0
  64. package/dist/registry.js +4 -0
  65. package/dist/resources/model-fit/assess.d.ts +37 -2
  66. package/dist/resources/model-fit/assess.js +24 -3
  67. package/dist/resources/model-fit/estimators/llm.js +2 -2
  68. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  69. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
  70. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
  71. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  72. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  73. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  74. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  75. package/dist/resources/model-fit/floor.js +2 -2
  76. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  77. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  78. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  79. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  80. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  81. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  82. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  83. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  84. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  85. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  86. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  87. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  88. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  89. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  90. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  91. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  92. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  93. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  94. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  95. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  96. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  97. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  98. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  99. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  100. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  101. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  102. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  103. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  104. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  105. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  106. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  107. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  108. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  109. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  110. package/dist/resources/model-fit/platform.d.ts +9 -0
  111. package/dist/resources/model-fit/platform.js +28 -0
  112. package/dist/rpc/device-map.d.ts +13 -0
  113. package/dist/rpc/device-map.js +22 -0
  114. package/dist/rpc/discovery.d.ts +12 -0
  115. package/dist/rpc/discovery.js +225 -0
  116. package/dist/rpc/ggml-provider.d.ts +4 -0
  117. package/dist/rpc/ggml-provider.js +25 -0
  118. package/dist/rpc/index.d.ts +2 -0
  119. package/dist/rpc/index.js +1 -0
  120. package/dist/rpc/instance.d.ts +6 -0
  121. package/dist/rpc/instance.js +21 -0
  122. package/dist/rpc/inventory.d.ts +5 -0
  123. package/dist/rpc/inventory.js +144 -0
  124. package/dist/rpc/manager.d.ts +14 -0
  125. package/dist/rpc/manager.js +174 -0
  126. package/dist/rpc/network.d.ts +11 -0
  127. package/dist/rpc/network.js +85 -0
  128. package/dist/rpc/provider.d.ts +6 -0
  129. package/dist/rpc/provider.js +24 -0
  130. package/dist/runtime/lifecycle.js +31 -13
  131. package/dist/runtime/request-context.d.ts +1 -1
  132. package/dist/runtime/state.js +5 -0
  133. package/dist/schemas/assess-model-fit.d.ts +269 -85
  134. package/dist/schemas/assess-model-fit.js +78 -54
  135. package/dist/schemas/audio-gen.d.ts +6 -6
  136. package/dist/schemas/audio-gen.js +2 -2
  137. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  138. package/dist/schemas/batch-completion-stream.js +5 -3
  139. package/dist/schemas/bci-config.js +1 -1
  140. package/dist/schemas/bci.d.ts +86 -1
  141. package/dist/schemas/bci.js +21 -2
  142. package/dist/schemas/cancel.d.ts +10 -0
  143. package/dist/schemas/cancel.js +2 -0
  144. package/dist/schemas/common.d.ts +708 -66
  145. package/dist/schemas/common.js +7 -0
  146. package/dist/schemas/completion-event.d.ts +6 -6
  147. package/dist/schemas/completion-stream.d.ts +27 -8
  148. package/dist/schemas/completion-stream.js +41 -4
  149. package/dist/schemas/config.d.ts +104 -36
  150. package/dist/schemas/config.js +10 -0
  151. package/dist/schemas/embed.d.ts +2 -2
  152. package/dist/schemas/error.js +13 -0
  153. package/dist/schemas/errors.d.ts +1 -0
  154. package/dist/schemas/errors.js +5 -0
  155. package/dist/schemas/index.d.ts +1 -0
  156. package/dist/schemas/index.js +1 -0
  157. package/dist/schemas/llamacpp-config.d.ts +25 -18
  158. package/dist/schemas/llamacpp-config.js +26 -16
  159. package/dist/schemas/load-model.d.ts +2608 -469
  160. package/dist/schemas/load-model.js +2 -2
  161. package/dist/schemas/mcp-adapter.d.ts +4 -0
  162. package/dist/schemas/model-config-schemas.d.ts +183 -6
  163. package/dist/schemas/model-info.d.ts +56 -0
  164. package/dist/schemas/model-info.js +5 -1
  165. package/dist/schemas/plugin.d.ts +8 -2
  166. package/dist/schemas/profiling.d.ts +4 -4
  167. package/dist/schemas/reload-config.d.ts +30 -0
  168. package/dist/schemas/rpc-server.d.ts +101 -0
  169. package/dist/schemas/rpc-server.js +87 -0
  170. package/dist/schemas/sdcpp-config.d.ts +7 -3
  171. package/dist/schemas/sdcpp-config.js +37 -2
  172. package/dist/schemas/system-resources.d.ts +4 -4
  173. package/dist/schemas/text-to-speech.d.ts +529 -1
  174. package/dist/schemas/text-to-speech.js +122 -12
  175. package/dist/schemas/tools.d.ts +2 -0
  176. package/dist/schemas/tools.js +8 -0
  177. package/dist/schemas/transcription-config.d.ts +5 -0
  178. package/dist/schemas/transcription-config.js +20 -2
  179. package/dist/schemas/transcription.d.ts +164 -0
  180. package/dist/schemas/transcription.js +71 -4
  181. package/dist/schemas/vla.d.ts +1 -1
  182. package/dist/surface.d.ts +4 -1
  183. package/dist/surface.js +3 -1
  184. package/dist/utils/addon-responses.d.ts +5 -1
  185. package/dist/utils/archive.js +5 -1
  186. package/dist/utils/asr-diagnostics.d.ts +12 -0
  187. package/dist/utils/asr-diagnostics.js +42 -0
  188. package/dist/utils/asr-events.d.ts +1 -0
  189. package/dist/utils/asr-events.js +3 -2
  190. package/dist/utils/cache/paths.js +3 -2
  191. package/dist/utils/mcp-adapter.js +7 -1
  192. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  193. package/dist/utils/multi-gpu-mobile.js +8 -6
  194. package/dist/utils/path-security.d.ts +5 -0
  195. package/dist/utils/path-security.js +14 -0
  196. package/dist/utils/tool-helpers.d.ts +4 -0
  197. package/dist/utils/tool-helpers.js +2 -0
  198. package/dist/utils/tools/defer.d.ts +93 -0
  199. package/dist/utils/tools/defer.js +290 -0
  200. package/dist/utils/tools/index.d.ts +1 -0
  201. package/dist/utils/tools/index.js +1 -0
  202. package/dist/utils/transcribe-metadata.d.ts +21 -7
  203. package/dist/utils/transcribe-metadata.js +24 -8
  204. package/dist/utils/tts-stats.d.ts +6 -0
  205. package/dist/utils/tts-stats.js +14 -1
  206. package/package.json +43 -29
  207. package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
  208. package/dist/resources/model-fit/fit-stub/index.js +0 -2
  209. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
  210. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
  211. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
@@ -12,8 +12,13 @@ import { getEngineLogger } from '../../../logging/index.js';
12
12
  * capped. Work still in flight at expiry is abandoned: the download is
13
13
  * aborted, and a lookup that cannot be cancelled settles on its own with
14
14
  * nothing waiting for it.
15
+ *
16
+ * One budget covers a whole companion set, so it holds the slowest case: every
17
+ * member of a multi-stage voice model staged over a mobile network. It sits
18
+ * above the client's per-attempt timeout rather than on it, so an attempt that
19
+ * is about to answer is not cut at the moment it would have.
15
20
  */
16
- export const FIT_STUB_BUDGET_MS = 10_000;
21
+ export const FIT_STUB_BUDGET_MS = 40_000;
17
22
  async function defaultGetEntry(registryPath, registrySource) {
18
23
  const { getRegistryClient } = await import('../../../runtime/registry-client.js');
19
24
  const client = await getRegistryClient();
@@ -56,6 +61,18 @@ function describe(error) {
56
61
  * llama's shard naming to read any of it — assembling that is not done here.
57
62
  */
58
63
  export async function fetchFitStub(ref, options = {}) {
64
+ return withinBudget(ref.name, options, (signal) => fetchWithin(ref, options, signal));
65
+ }
66
+ /**
67
+ * Stages a companion set under its target names in one directory, the layout a
68
+ * downloaded set has, for engines that find their companions beside the
69
+ * checkpoint. The primary decides the outcome; a member the registry holds no
70
+ * description for is skipped.
71
+ */
72
+ export async function fetchFitStubSet(set, options = {}) {
73
+ return withinBudget(set.primary.name, options, (signal) => fetchSetWithin(set, options, signal));
74
+ }
75
+ function withinBudget(name, options, work) {
59
76
  const budgetMs = options.budgetMs ?? FIT_STUB_BUDGET_MS;
60
77
  const controller = new AbortController();
61
78
  const onAbort = () => controller.abort(new Error('fit stub fetch aborted by the caller'));
@@ -64,70 +81,132 @@ export async function fetchFitStub(ref, options = {}) {
64
81
  const expiry = new Promise((resolve) => {
65
82
  timer = setTimeout(() => {
66
83
  controller.abort(new Error('fit stub budget expired'));
67
- resolve(unavailable('timed-out', `no fit stub for ${ref.name} from the registry within ${budgetMs}ms`));
84
+ resolve(unavailable('timed-out', `no fit stub for ${name} from the registry within ${budgetMs}ms`));
68
85
  }, budgetMs);
69
86
  });
87
+ return Promise.race([work(controller.signal), expiry]).finally(() => {
88
+ clearTimeout(timer);
89
+ options.signal?.removeEventListener('abort', onAbort);
90
+ });
91
+ }
92
+ // One directory per fetch, so two assessments of the same model never share a
93
+ // file that one of them is about to remove.
94
+ async function makeStagingDir(options) {
95
+ const root = options.cacheDir ?? getCacheDir('fit-stubs');
96
+ await fsPromises.mkdir(root, { recursive: true });
97
+ return fsPromises.mkdtemp(path.join(root, 'stub-'));
98
+ }
99
+ async function bindingFor(ref, options) {
100
+ const { registryPath, registrySource } = ref;
101
+ if (!registryPath || !registrySource)
102
+ return 'unresolvable-ref';
103
+ const getEntry = options.getEntry ?? defaultGetEntry;
104
+ const entry = await getEntry(registryPath, registrySource);
105
+ if (entry === null || entry === undefined)
106
+ return 'not-in-registry';
107
+ const binding = entry.fitBlobBinding;
108
+ if (binding === undefined || binding === null)
109
+ return 'no-fit-blob';
110
+ return binding;
111
+ }
112
+ async function stageOne(ref, binding, dest, options, signal) {
113
+ const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
114
+ let verified = false;
70
115
  try {
71
- return await Promise.race([fetchWithin(ref, options, controller.signal), expiry]);
116
+ await downloadBlob(binding, dest, signal);
117
+ const bytes = fs.statSync(dest).size;
118
+ if (bytes !== binding.byteLength) {
119
+ return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
120
+ }
121
+ // The record binds the description by digest; a stub that reads back
122
+ // differently is not the description the fitter should answer for.
123
+ const digest = await calculateFileChecksum(dest);
124
+ if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
125
+ return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
126
+ }
127
+ // Past the budget nobody is waiting for this result, so the stub must
128
+ // not be marked as kept.
129
+ if (signal.aborted)
130
+ return unavailable('timed-out');
131
+ verified = true;
132
+ return { status: 'staged', bytes };
133
+ }
134
+ catch (error) {
135
+ return unavailable('download-failed', describe(error));
72
136
  }
73
137
  finally {
74
- clearTimeout(timer);
75
- options.signal?.removeEventListener('abort', onAbort);
138
+ // A short, corrupt, half-written or late stub must not outlive the call.
139
+ if (!verified)
140
+ await fsPromises.rm(dest, { force: true }).catch(() => { });
76
141
  }
77
142
  }
78
143
  async function fetchWithin(ref, options, signal) {
79
144
  const logger = options.logger ?? getEngineLogger();
80
145
  try {
81
- const { registryPath, registrySource } = ref;
82
- if (!registryPath || !registrySource)
83
- return unavailable('unresolvable-ref');
84
- const getEntry = options.getEntry ?? defaultGetEntry;
85
- const entry = await getEntry(registryPath, registrySource);
86
- if (entry === null || entry === undefined)
87
- return unavailable('not-in-registry');
88
- const binding = entry.fitBlobBinding;
89
- if (binding === undefined || binding === null)
90
- return unavailable('no-fit-blob');
146
+ const binding = await bindingFor(ref, options);
147
+ if (typeof binding === 'string')
148
+ return unavailable(binding);
91
149
  if (signal.aborted)
92
150
  return unavailable('timed-out');
93
- // One directory per fetch, so two assessments of the same model never share
94
- // a file that one of them is about to remove.
95
- const root = options.cacheDir ?? getCacheDir('fit-stubs');
96
- await fsPromises.mkdir(root, { recursive: true });
97
- const dir = await fsPromises.mkdtemp(path.join(root, 'stub-'));
151
+ const dir = await makeStagingDir(options);
98
152
  const dest = path.join(dir, `${binding.sha256}.gguf`);
99
- const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
100
- let verified = false;
101
- try {
102
- await downloadBlob(binding, dest, signal);
103
- const bytes = fs.statSync(dest).size;
104
- if (bytes !== binding.byteLength) {
105
- return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
153
+ const staged = await stageOne(ref, binding, dest, options, signal);
154
+ if (staged.status !== 'staged') {
155
+ await removeStub(dest);
156
+ return staged;
157
+ }
158
+ return { status: 'ready', path: dest, bytes: staged.bytes };
159
+ }
160
+ catch (error) {
161
+ logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
162
+ return unavailable('download-failed', describe(error));
163
+ }
164
+ }
165
+ async function fetchSetWithin(set, options, signal) {
166
+ const logger = options.logger ?? getEngineLogger();
167
+ let dir;
168
+ let ready = false;
169
+ try {
170
+ const binding = await bindingFor(set.primary, options);
171
+ if (typeof binding === 'string')
172
+ return unavailable(binding);
173
+ if (signal.aborted)
174
+ return unavailable('timed-out');
175
+ dir = await makeStagingDir(options);
176
+ const dest = path.join(dir, set.primary.targetName);
177
+ const staged = await stageOne(set.primary, binding, dest, options, signal);
178
+ if (staged.status !== 'staged')
179
+ return staged;
180
+ for (const member of set.others) {
181
+ if (signal.aborted)
182
+ break;
183
+ const memberBinding = await bindingFor(member, options);
184
+ if (typeof memberBinding === 'string') {
185
+ logger.debug(`companion ${member.targetName} has no fit stub: ${memberBinding}`);
186
+ continue;
106
187
  }
107
- // The record binds the description by digest; a stub that reads back
108
- // differently is not the description the fitter should answer for.
109
- const digest = await calculateFileChecksum(dest);
110
- if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
111
- return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
188
+ const memberDest = path.join(dir, member.targetName);
189
+ const memberStaged = await stageOne(member, memberBinding, memberDest, options, signal);
190
+ if (memberStaged.status !== 'staged') {
191
+ logger.debug(`companion ${member.targetName} was not staged`);
112
192
  }
113
- // Past the budget nobody is waiting for this result, so the stub must
114
- // not be marked as kept.
115
- if (signal.aborted)
116
- return unavailable('timed-out');
117
- verified = true;
118
- return { status: 'ready', path: dest, bytes };
119
- }
120
- catch (error) {
121
- return unavailable('download-failed', describe(error));
122
- }
123
- finally {
124
- // A short, corrupt, half-written or late stub must not outlive the call.
125
- if (!verified)
126
- await removeStub(dest);
127
193
  }
194
+ // The budget races this work and has already answered the caller once it
195
+ // expires, so a set finished after that point is staged for nobody.
196
+ // The budget answers the caller on its own, so a set finished after it
197
+ // expired is staged for nobody.
198
+ if (signal.aborted)
199
+ return unavailable('timed-out');
200
+ ready = true;
201
+ return { status: 'ready', path: dest, bytes: staged.bytes };
128
202
  }
129
203
  catch (error) {
130
- logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
204
+ logger.debug(`fit stub set for ${set.primary.name} unavailable: ${describe(error)}`);
131
205
  return unavailable('download-failed', describe(error));
132
206
  }
207
+ finally {
208
+ if (!ready && dir !== undefined) {
209
+ await fsPromises.rm(dir, { recursive: true, force: true }).catch(() => { });
210
+ }
211
+ }
133
212
  }
@@ -0,0 +1,27 @@
1
+ import { type LoadDescription } from './resolve-load-from-stubs.js';
2
+ import type { FitStubOptions, FitStubUnavailableReason } from './fetch-fit-stub.js';
3
+ import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
4
+ export type LoadFitOutcome = {
5
+ status: 'projected';
6
+ fit: AdvisoryFitOutcome;
7
+ } | {
8
+ status: 'no-stub';
9
+ reason: FitStubUnavailableReason;
10
+ message?: string;
11
+ } | {
12
+ status: 'unsupported-load';
13
+ detail: string;
14
+ };
15
+ export interface LoadFitOptions {
16
+ stub?: FitStubOptions;
17
+ fit?: AdvisoryFitOptions;
18
+ }
19
+ /**
20
+ * Projects the load a caller describes, before its weights exist locally.
21
+ *
22
+ * The same probe `loadModel` runs, over the same config and artifacts that
23
+ * load's own plugin resolves, pointed at files that are tens of KB instead of
24
+ * gigabytes. It inherits the probe's fail-open contract: every failure is an
25
+ * `unknown` verdict or a non-projected outcome, never a throw.
26
+ */
27
+ export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
@@ -0,0 +1,31 @@
1
+ import { resolveLoadFromStubs } from './resolve-load-from-stubs.js';
2
+ import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
3
+ /**
4
+ * Projects the load a caller describes, before its weights exist locally.
5
+ *
6
+ * The same probe `loadModel` runs, over the same config and artifacts that
7
+ * load's own plugin resolves, pointed at files that are tens of KB instead of
8
+ * gigabytes. It inherits the probe's fail-open contract: every failure is an
9
+ * `unknown` verdict or a non-projected outcome, never a throw.
10
+ */
11
+ export async function projectFitFromLoad(load, modelId, options = {}) {
12
+ const resolved = await resolveLoadFromStubs(load, options.stub);
13
+ if (resolved.status !== 'resolved')
14
+ return resolved;
15
+ try {
16
+ const fit = await runAdvisoryFitCheck({
17
+ modelId,
18
+ modelType: load.modelType,
19
+ modelPath: resolved.modelPath,
20
+ modelConfig: resolved.modelConfig,
21
+ artifacts: resolved.artifacts,
22
+ // Nothing here assembles a shard set, so no load may claim the fitter
23
+ // is looking at a split model.
24
+ isShardedModel: false
25
+ }, options.fit);
26
+ return { status: 'projected', fit };
27
+ }
28
+ finally {
29
+ await resolved.release();
30
+ }
31
+ }
@@ -0,0 +1,38 @@
1
+ import type { ModelSrcInput } from '../../../schemas/index.js';
2
+ import type { CanonicalModelType } from '../../../schemas/index.js';
3
+ import { type FitStubOptions, type FitStubUnavailableReason } from './fetch-fit-stub.js';
4
+ export interface LoadDescription {
5
+ /** Absent for the loads `loadModel` also takes without one. */
6
+ modelSrc?: ModelSrcInput | undefined;
7
+ modelType: CanonicalModelType;
8
+ modelConfig?: Record<string, unknown> | undefined;
9
+ }
10
+ export type ResolvedLoad = {
11
+ status: 'resolved';
12
+ modelPath: string;
13
+ modelConfig: unknown;
14
+ artifacts: Record<string, string>;
15
+ /** Removes every stub staged for this load. Never throws. */
16
+ release: () => Promise<void>;
17
+ };
18
+ export type LoadResolution = ResolvedLoad | {
19
+ status: 'no-stub';
20
+ reason: FitStubUnavailableReason;
21
+ message?: string;
22
+ } | {
23
+ status: 'unsupported-load';
24
+ detail: string;
25
+ };
26
+ /**
27
+ * Resolves a load the way `loadModel` would, against the registry's weightless
28
+ * descriptions instead of the weights.
29
+ *
30
+ * The plugin's own `resolveConfig` does the work: it strips the source fields
31
+ * from the config and returns the artifact paths keyed the way its engine
32
+ * expects. Only what a source resolves to differs here, so the config and the
33
+ * keys are the real load's.
34
+ *
35
+ * A source already on disk is read directly, since it describes the load more
36
+ * exactly than a stub does.
37
+ */
38
+ export declare function resolveLoadFromStubs(load: LoadDescription, options?: FitStubOptions): Promise<LoadResolution>;
@@ -0,0 +1,138 @@
1
+ import fs from 'bare-fs';
2
+ import { getPlugin } from '../../../plugins/registry.js';
3
+ import { getRuntimeContext } from '../../../runtime/state.js';
4
+ import { formatZodError } from '../../../utils/zod-error.js';
5
+ import { getModelByPath } from '../../../models/registry/index.js';
6
+ import { fetchFitStub, fetchFitStubSet, removeStub } from './fetch-fit-stub.js';
7
+ function srcString(modelSrc) {
8
+ return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
9
+ }
10
+ /**
11
+ * The stub ref a source carries, or `undefined` where it names no registry
12
+ * artifact. A catalog constant carries the checksum and coordinates; a bare
13
+ * string names a location and nothing the registry can be asked about.
14
+ */
15
+ function stubRefFor(modelSrc) {
16
+ if (typeof modelSrc === 'string')
17
+ return undefined;
18
+ const { sha256Checksum, registryPath, registrySource } = modelSrc;
19
+ if (sha256Checksum === undefined || registryPath === undefined)
20
+ return undefined;
21
+ return {
22
+ name: modelSrc.name ?? modelSrc.modelId ?? registryPath,
23
+ sha256Checksum,
24
+ registryPath,
25
+ ...(registrySource !== undefined && { registrySource })
26
+ };
27
+ }
28
+ function stubSetFor(registryPath) {
29
+ const companionSet = getModelByPath(registryPath)?.companionSet;
30
+ if (companionSet === undefined)
31
+ return undefined;
32
+ let primary;
33
+ const others = [];
34
+ for (const file of companionSet.files) {
35
+ const member = {
36
+ name: file.targetName,
37
+ sha256Checksum: file.sha256Checksum,
38
+ registryPath: file.registryPath,
39
+ registrySource: file.registrySource,
40
+ targetName: file.targetName
41
+ };
42
+ if (file.key === companionSet.primaryKey)
43
+ primary = member;
44
+ else
45
+ others.push(member);
46
+ }
47
+ return primary === undefined ? undefined : { primary, others };
48
+ }
49
+ class NoStubError extends Error {
50
+ reason;
51
+ constructor(reason, message) {
52
+ super(message ?? reason);
53
+ this.reason = reason;
54
+ }
55
+ }
56
+ /**
57
+ * Resolves a load the way `loadModel` would, against the registry's weightless
58
+ * descriptions instead of the weights.
59
+ *
60
+ * The plugin's own `resolveConfig` does the work: it strips the source fields
61
+ * from the config and returns the artifact paths keyed the way its engine
62
+ * expects. Only what a source resolves to differs here, so the config and the
63
+ * keys are the real load's.
64
+ *
65
+ * A source already on disk is read directly, since it describes the load more
66
+ * exactly than a stub does.
67
+ */
68
+ export async function resolveLoadFromStubs(load, options = {}) {
69
+ const plugin = getPlugin(load.modelType);
70
+ if (!plugin) {
71
+ return { status: 'unsupported-load', detail: `no plugin registered for ${load.modelType}` };
72
+ }
73
+ const parsed = plugin.loadConfigSchema.safeParse(load.modelConfig ?? {});
74
+ if (!parsed.success) {
75
+ return { status: 'unsupported-load', detail: formatZodError(parsed.error) };
76
+ }
77
+ const modelConfig = parsed.data;
78
+ const staged = [];
79
+ const release = async () => {
80
+ for (const path of staged)
81
+ await removeStub(path);
82
+ };
83
+ const resolveModelPath = async (src) => {
84
+ const location = srcString(src);
85
+ if (location !== '' && fs.existsSync(location))
86
+ return location;
87
+ const ref = stubRefFor(src);
88
+ if (ref === undefined)
89
+ throw new NoStubError('unresolvable-ref', location);
90
+ const set = stubSetFor(ref.registryPath);
91
+ const stub = set === undefined ? await fetchFitStub(ref, options) : await fetchFitStubSet(set, options);
92
+ if (stub.status !== 'ready')
93
+ throw new NoStubError(stub.reason, stub.message);
94
+ staged.push(stub.path);
95
+ return stub.path;
96
+ };
97
+ try {
98
+ const modelPath = load.modelSrc === undefined ? '' : await resolveModelPath(load.modelSrc);
99
+ if (plugin.resolveConfig === undefined) {
100
+ return {
101
+ status: 'resolved',
102
+ modelPath,
103
+ modelConfig,
104
+ artifacts: {},
105
+ release
106
+ };
107
+ }
108
+ const context = {
109
+ resolveModelPath,
110
+ modelSrc: load.modelSrc === undefined ? '' : srcString(load.modelSrc),
111
+ modelType: load.modelType,
112
+ ...(getRuntimeContext().platform !== undefined && {
113
+ platform: getRuntimeContext().platform
114
+ })
115
+ };
116
+ const resolved = await plugin.resolveConfig(modelConfig, context);
117
+ return {
118
+ status: 'resolved',
119
+ modelPath,
120
+ modelConfig: resolved.config,
121
+ artifacts: (resolved.artifacts ?? {}),
122
+ release
123
+ };
124
+ }
125
+ catch (error) {
126
+ await release();
127
+ if (error instanceof NoStubError) {
128
+ return error.message === error.reason
129
+ ? { status: 'no-stub', reason: error.reason }
130
+ : { status: 'no-stub', reason: error.reason, message: error.message };
131
+ }
132
+ // A config the plugin refuses describes a load that would not start either.
133
+ return {
134
+ status: 'unsupported-load',
135
+ detail: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
136
+ };
137
+ }
138
+ }
@@ -47,7 +47,7 @@ export function computeFloor(input) {
47
47
  reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
48
48
  return { bytes: artifactBytes, reasons, assumptions };
49
49
  }
50
- let contextTokens = workload.contextTokens;
50
+ let contextTokens = workload.contextTokens ?? facts.contextLength;
51
51
  if (contextTokens > facts.contextLength) {
52
52
  assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
53
53
  contextTokens = facts.contextLength;
@@ -59,7 +59,7 @@ export function computeFloor(input) {
59
59
  const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
60
60
  assumptions.push(element < 2
61
61
  ? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
62
- : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not expressible in a workload and would change these numbers');
62
+ : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
63
63
  reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
64
64
  return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
65
65
  }
@@ -1,12 +1,12 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
2
  import type { Logger } from '../../../logging/types.js';
3
3
  import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
4
- import type { runIsolatedFit } from './run-isolated-fit.js';
4
+ import { runFit as runFitDefault } from './run-fit.js';
5
5
  /**
6
6
  * `fit` and `does-not-fit` are projections of the load the SDK is about to run,
7
- * not admission decisions. `@qvac/model-fit` duplicates the loader's policy for
8
- * this experiment and the real loader neither consumes nor verifies the fitted
9
- * plan, so neither verdict is denial-grade and no verdict changes the load.
7
+ * not admission decisions. The real loader neither consumes nor verifies the
8
+ * fitted plan, so neither verdict is denial-grade and no verdict changes the
9
+ * load.
10
10
  *
11
11
  * The outcome is the wire shape: `loadModel` hands it to the model registry and
12
12
  * `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
@@ -31,12 +31,15 @@ export interface AdvisoryFitOptions {
31
31
  enabled?: boolean;
32
32
  mobile?: boolean;
33
33
  timeoutMs?: number;
34
- runFit?: typeof runIsolatedFit;
34
+ runFit?: typeof runFitDefault;
35
35
  logger?: Logger;
36
+ availableSystemBytes?: () => Promise<number | undefined>;
37
+ countsDeviceRows?: () => Promise<boolean>;
36
38
  residentModelBytes?: () => Promise<number>;
37
39
  }
38
40
  /**
39
- * Runs the advisory llama.cpp fit check for a load that is about to start.
41
+ * Runs the advisory fit check for a load that is about to start, against the
42
+ * fitter belonging to the engine that will run it.
40
43
  *
41
44
  * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
42
45
  * malformed response, and an unexpected internal error all resolve to `unknown`