@qvac/inference 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/NOTICE +107 -33
  2. package/README.md +8 -5
  3. package/dist/api/assess-model-fit.d.ts +9 -4
  4. package/dist/api/assess-model-fit.js +9 -4
  5. package/dist/api/bci-transcribe.d.ts +4 -1
  6. package/dist/api/index.d.ts +1 -0
  7. package/dist/api/index.js +1 -0
  8. package/dist/api/rpc-server.d.ts +19 -0
  9. package/dist/api/rpc-server.js +44 -0
  10. package/dist/api/text-to-speech.d.ts +9 -0
  11. package/dist/api/text-to-speech.js +9 -0
  12. package/dist/api/transcribe.d.ts +11 -7
  13. package/dist/api/transcribe.js +47 -15
  14. package/dist/api/video-request.d.ts +5 -5
  15. package/dist/dispatch.js +39 -7
  16. package/dist/errors/index.d.ts +7 -0
  17. package/dist/errors/index.js +15 -0
  18. package/dist/handlers/assess-model-fit.d.ts +18 -6
  19. package/dist/handlers/assess-model-fit.js +215 -35
  20. package/dist/handlers/completion-orchestrate.js +17 -0
  21. package/dist/handlers/get-loaded-model-info.js +4 -1
  22. package/dist/handlers/get-model-info.js +29 -1
  23. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  24. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  25. package/dist/handlers/load-model/hyperdrive.js +2 -2
  26. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  27. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  28. package/dist/handlers/load-model/registry.js +59 -0
  29. package/dist/handlers/rpc-server.d.ts +23 -0
  30. package/dist/handlers/rpc-server.js +141 -0
  31. package/dist/index.d.ts +2 -1
  32. package/dist/index.js +2 -1
  33. package/dist/models/registry/models.d.ts +7731 -5672
  34. package/dist/models/registry/models.js +4652 -2526
  35. package/dist/models/registry/resource-profiles.js +481 -94
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  40. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  44. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
  45. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  46. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  47. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  48. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  49. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  50. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  51. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  52. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  54. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  55. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  56. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  57. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  58. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  59. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  60. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  61. package/dist/plugins/ops/bci-transcribe.js +18 -13
  62. package/dist/plugins/ops/load-model.js +2 -2
  63. package/dist/plugins/ops/transcribe.d.ts +4 -2
  64. package/dist/plugins/ops/transcribe.js +39 -2
  65. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  66. package/dist/plugins/ops/translate-stats.js +64 -7
  67. package/dist/plugins/ops/translate.js +20 -4
  68. package/dist/profiling/operation-metrics.js +56 -0
  69. package/dist/registry.js +4 -0
  70. package/dist/resources/model-fit/assess.d.ts +46 -2
  71. package/dist/resources/model-fit/assess.js +67 -3
  72. package/dist/resources/model-fit/estimators/llm.js +2 -2
  73. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  74. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
  75. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
  76. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  77. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  78. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  79. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  80. package/dist/resources/model-fit/floor.js +2 -2
  81. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  82. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  83. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  84. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  85. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  86. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  87. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  88. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  89. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  90. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  91. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  92. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  93. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  94. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  95. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  96. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  97. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  98. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  99. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  100. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  101. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  102. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  103. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  104. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  105. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  106. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  107. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  108. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  109. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  110. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  111. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  112. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  113. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  114. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  115. package/dist/resources/model-fit/platform.d.ts +9 -0
  116. package/dist/resources/model-fit/platform.js +28 -0
  117. package/dist/rpc/device-map.d.ts +13 -0
  118. package/dist/rpc/device-map.js +22 -0
  119. package/dist/rpc/discovery.d.ts +12 -0
  120. package/dist/rpc/discovery.js +225 -0
  121. package/dist/rpc/ggml-provider.d.ts +4 -0
  122. package/dist/rpc/ggml-provider.js +25 -0
  123. package/dist/rpc/index.d.ts +2 -0
  124. package/dist/rpc/index.js +1 -0
  125. package/dist/rpc/instance.d.ts +6 -0
  126. package/dist/rpc/instance.js +21 -0
  127. package/dist/rpc/inventory.d.ts +5 -0
  128. package/dist/rpc/inventory.js +144 -0
  129. package/dist/rpc/manager.d.ts +14 -0
  130. package/dist/rpc/manager.js +174 -0
  131. package/dist/rpc/network.d.ts +11 -0
  132. package/dist/rpc/network.js +85 -0
  133. package/dist/rpc/provider.d.ts +6 -0
  134. package/dist/rpc/provider.js +24 -0
  135. package/dist/runtime/lifecycle.js +31 -13
  136. package/dist/runtime/request-context.d.ts +1 -1
  137. package/dist/runtime/state.js +5 -0
  138. package/dist/schemas/assess-model-fit.d.ts +284 -77
  139. package/dist/schemas/assess-model-fit.js +93 -55
  140. package/dist/schemas/audio-gen.d.ts +6 -6
  141. package/dist/schemas/audio-gen.js +2 -2
  142. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  143. package/dist/schemas/batch-completion-stream.js +5 -3
  144. package/dist/schemas/bci-config.js +1 -1
  145. package/dist/schemas/bci.d.ts +86 -1
  146. package/dist/schemas/bci.js +21 -2
  147. package/dist/schemas/cancel.d.ts +10 -0
  148. package/dist/schemas/cancel.js +2 -0
  149. package/dist/schemas/common.d.ts +765 -56
  150. package/dist/schemas/common.js +7 -0
  151. package/dist/schemas/completion-event.d.ts +6 -6
  152. package/dist/schemas/completion-stream.d.ts +27 -8
  153. package/dist/schemas/completion-stream.js +41 -4
  154. package/dist/schemas/config.d.ts +236 -30
  155. package/dist/schemas/config.js +10 -0
  156. package/dist/schemas/embed.d.ts +2 -2
  157. package/dist/schemas/error.js +13 -0
  158. package/dist/schemas/errors.d.ts +1 -0
  159. package/dist/schemas/errors.js +5 -0
  160. package/dist/schemas/index.d.ts +1 -0
  161. package/dist/schemas/index.js +1 -0
  162. package/dist/schemas/llamacpp-config.d.ts +105 -14
  163. package/dist/schemas/llamacpp-config.js +113 -3
  164. package/dist/schemas/load-model.d.ts +3102 -627
  165. package/dist/schemas/load-model.js +2 -2
  166. package/dist/schemas/mcp-adapter.d.ts +4 -0
  167. package/dist/schemas/model-config-schemas.d.ts +205 -5
  168. package/dist/schemas/model-info.d.ts +56 -0
  169. package/dist/schemas/model-info.js +5 -1
  170. package/dist/schemas/plugin.d.ts +8 -2
  171. package/dist/schemas/profiling.d.ts +4 -4
  172. package/dist/schemas/reload-config.d.ts +30 -0
  173. package/dist/schemas/rpc-server.d.ts +101 -0
  174. package/dist/schemas/rpc-server.js +87 -0
  175. package/dist/schemas/sdcpp-config.d.ts +7 -3
  176. package/dist/schemas/sdcpp-config.js +37 -2
  177. package/dist/schemas/system-resources.d.ts +4 -4
  178. package/dist/schemas/text-to-speech.d.ts +529 -1
  179. package/dist/schemas/text-to-speech.js +122 -12
  180. package/dist/schemas/tools.d.ts +2 -0
  181. package/dist/schemas/tools.js +8 -0
  182. package/dist/schemas/transcription-config.d.ts +5 -0
  183. package/dist/schemas/transcription-config.js +20 -2
  184. package/dist/schemas/transcription.d.ts +164 -0
  185. package/dist/schemas/transcription.js +71 -4
  186. package/dist/schemas/vla.d.ts +1 -1
  187. package/dist/surface.d.ts +4 -1
  188. package/dist/surface.js +3 -1
  189. package/dist/utils/addon-responses.d.ts +5 -1
  190. package/dist/utils/archive.js +5 -1
  191. package/dist/utils/asr-diagnostics.d.ts +12 -0
  192. package/dist/utils/asr-diagnostics.js +42 -0
  193. package/dist/utils/asr-events.d.ts +1 -0
  194. package/dist/utils/asr-events.js +3 -2
  195. package/dist/utils/cache/paths.js +3 -2
  196. package/dist/utils/mcp-adapter.js +7 -1
  197. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  198. package/dist/utils/multi-gpu-mobile.js +8 -6
  199. package/dist/utils/path-security.d.ts +5 -0
  200. package/dist/utils/path-security.js +14 -0
  201. package/dist/utils/tool-helpers.d.ts +4 -0
  202. package/dist/utils/tool-helpers.js +2 -0
  203. package/dist/utils/tools/defer.d.ts +93 -0
  204. package/dist/utils/tools/defer.js +290 -0
  205. package/dist/utils/tools/index.d.ts +1 -0
  206. package/dist/utils/tools/index.js +1 -0
  207. package/dist/utils/transcribe-metadata.d.ts +21 -7
  208. package/dist/utils/transcribe-metadata.js +24 -8
  209. package/dist/utils/tts-stats.d.ts +6 -0
  210. package/dist/utils/tts-stats.js +14 -1
  211. package/package.json +44 -30
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  213. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
@@ -0,0 +1,102 @@
1
+ import { type AbortSignal } from 'bare-abort-controller';
2
+ import type { QVACBlobBinding } from '@qvac/registry-client';
3
+ import type { Logger } from '../../../logging/types.js';
4
+ /**
5
+ * A weightless description of an artifact, served by the registry beside the
6
+ * weights: for a GGUF, the tensor list and the settings with the tokenizer
7
+ * tables stripped and no data section — tens of KB against gigabytes. The
8
+ * fitter reads it exactly as it reads the artifact, so a projection can be made
9
+ * before anything is downloaded.
10
+ */
11
+ export interface FitStubRef {
12
+ name: string;
13
+ sha256Checksum: string;
14
+ /** Registry coordinates. A constant that carries neither cannot be resolved. */
15
+ registryPath?: string | undefined;
16
+ registrySource?: string | undefined;
17
+ }
18
+ export interface FitStubSetMember extends FitStubRef {
19
+ targetName: string;
20
+ }
21
+ export interface FitStubSet {
22
+ primary: FitStubSetMember;
23
+ others: readonly FitStubSetMember[];
24
+ }
25
+ export type FitStubUnavailableReason =
26
+ /** The ref carries no registry coordinates to look up. */
27
+ 'unresolvable-ref'
28
+ /** The registry has no entry at those coordinates. */
29
+ | 'not-in-registry'
30
+ /** The entry predates fit blobs, or its description could not be built. */
31
+ | 'no-fit-blob' | 'download-failed'
32
+ /** The lookup and the fetch together did not finish within the budget. */
33
+ | 'timed-out';
34
+ export type FitStubOutcome =
35
+ /** `path` is the caller's to remove once the fitter has read it. */
36
+ {
37
+ status: 'ready';
38
+ path: string;
39
+ bytes: number;
40
+ } | {
41
+ status: 'unavailable';
42
+ reason: FitStubUnavailableReason;
43
+ message?: string;
44
+ };
45
+ export type FitBlobBinding = QVACBlobBinding;
46
+ export interface FitStubEntry {
47
+ fitBlobBinding?: FitBlobBinding | undefined;
48
+ }
49
+ /**
50
+ * Injection seams. Every field defaults to the real runtime dependency; tests
51
+ * substitute them rather than mocking modules.
52
+ */
53
+ export interface FitStubOptions {
54
+ signal?: AbortSignal;
55
+ /** Where stubs are staged. Defaults to `fit-stubs` under the QVAC cache root. */
56
+ cacheDir?: string;
57
+ /** Overall budget for the lookup and the fetch. Defaults to `FIT_STUB_BUDGET_MS`. */
58
+ budgetMs?: number;
59
+ logger?: Logger;
60
+ getEntry?: (registryPath: string, registrySource: string) => Promise<FitStubEntry | null>;
61
+ downloadBlob?: (binding: FitBlobBinding, outputFile: string, signal?: AbortSignal) => Promise<unknown>;
62
+ }
63
+ /**
64
+ * Bounds the registry lookup and the blob fetch together. The client's own
65
+ * defaults — 30s per attempt, three attempts, a peer wait between them, and an
66
+ * uncapped wait for the registry view on first use — suit a weights download.
67
+ * Here an unreachable registry is the whole latency of the call, so it is
68
+ * capped. Work still in flight at expiry is abandoned: the download is
69
+ * aborted, and a lookup that cannot be cancelled settles on its own with
70
+ * nothing waiting for it.
71
+ *
72
+ * One budget covers a whole companion set, so it holds the slowest case: every
73
+ * member of a multi-stage voice model staged over a mobile network. It sits
74
+ * above the client's per-attempt timeout rather than on it, so an attempt that
75
+ * is about to answer is not cut at the moment it would have.
76
+ */
77
+ export declare const FIT_STUB_BUDGET_MS = 40000;
78
+ /** Removes a fetched stub and the directory it was staged in. Never throws. */
79
+ export declare function removeStub(stubPath: string): Promise<void>;
80
+ /**
81
+ * Fetches the fit stub for one artifact into its own staging directory and
82
+ * returns its path. The stub is a per-call payload, not a cache: the caller
83
+ * removes it once the fitter has read it, so nothing accumulates under the
84
+ * QVAC root and no cleanup path has to know about it.
85
+ *
86
+ * Never throws: a missing entry, an entry without a description, a failed or
87
+ * corrupt download and an expired budget are all `unavailable`, because a
88
+ * pre-download assessment has to survive an offline caller and an older
89
+ * registry record.
90
+ *
91
+ * Only the artifact named by `ref` is fetched. A split model's shards each
92
+ * carry their own binding, and the fitter needs the whole set laid out under
93
+ * llama's shard naming to read any of it — assembling that is not done here.
94
+ */
95
+ export declare function fetchFitStub(ref: FitStubRef, options?: FitStubOptions): Promise<FitStubOutcome>;
96
+ /**
97
+ * Stages a companion set under its target names in one directory, the layout a
98
+ * downloaded set has, for engines that find their companions beside the
99
+ * checkpoint. The primary decides the outcome; a member the registry holds no
100
+ * description for is skipped.
101
+ */
102
+ export declare function fetchFitStubSet(set: FitStubSet, options?: FitStubOptions): Promise<FitStubOutcome>;
@@ -0,0 +1,212 @@
1
+ import fs, { promises as fsPromises } from 'bare-fs';
2
+ import path from 'bare-path';
3
+ import { AbortController } from 'bare-abort-controller';
4
+ import { getCacheDir } from '../../../utils/cache/paths.js';
5
+ import { calculateFileChecksum } from '../../../utils/checksum.js';
6
+ import { getEngineLogger } from '../../../logging/index.js';
7
+ /**
8
+ * Bounds the registry lookup and the blob fetch together. The client's own
9
+ * defaults — 30s per attempt, three attempts, a peer wait between them, and an
10
+ * uncapped wait for the registry view on first use — suit a weights download.
11
+ * Here an unreachable registry is the whole latency of the call, so it is
12
+ * capped. Work still in flight at expiry is abandoned: the download is
13
+ * aborted, and a lookup that cannot be cancelled settles on its own with
14
+ * nothing waiting for it.
15
+ *
16
+ * One budget covers a whole companion set, so it holds the slowest case: every
17
+ * member of a multi-stage voice model staged over a mobile network. It sits
18
+ * above the client's per-attempt timeout rather than on it, so an attempt that
19
+ * is about to answer is not cut at the moment it would have.
20
+ */
21
+ export const FIT_STUB_BUDGET_MS = 40_000;
22
+ async function defaultGetEntry(registryPath, registrySource) {
23
+ const { getRegistryClient } = await import('../../../runtime/registry-client.js');
24
+ const client = await getRegistryClient();
25
+ return client.getModel(registryPath, registrySource);
26
+ }
27
+ async function defaultDownloadBlob(binding, outputFile, signal) {
28
+ const { getRegistryClient } = await import('../../../runtime/registry-client.js');
29
+ const client = await getRegistryClient();
30
+ return client.downloadBlob(binding, {
31
+ outputFile,
32
+ maxRetries: 1,
33
+ ...(signal !== undefined && { signal })
34
+ });
35
+ }
36
+ /** Removes a fetched stub and the directory it was staged in. Never throws. */
37
+ export async function removeStub(stubPath) {
38
+ await fsPromises.rm(path.dirname(stubPath), { recursive: true, force: true }).catch(() => { });
39
+ }
40
+ function unavailable(reason, message) {
41
+ return message === undefined
42
+ ? { status: 'unavailable', reason }
43
+ : { status: 'unavailable', reason, message };
44
+ }
45
+ function describe(error) {
46
+ return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
47
+ }
48
+ /**
49
+ * Fetches the fit stub for one artifact into its own staging directory and
50
+ * returns its path. The stub is a per-call payload, not a cache: the caller
51
+ * removes it once the fitter has read it, so nothing accumulates under the
52
+ * QVAC root and no cleanup path has to know about it.
53
+ *
54
+ * Never throws: a missing entry, an entry without a description, a failed or
55
+ * corrupt download and an expired budget are all `unavailable`, because a
56
+ * pre-download assessment has to survive an offline caller and an older
57
+ * registry record.
58
+ *
59
+ * Only the artifact named by `ref` is fetched. A split model's shards each
60
+ * carry their own binding, and the fitter needs the whole set laid out under
61
+ * llama's shard naming to read any of it — assembling that is not done here.
62
+ */
63
+ export async function fetchFitStub(ref, options = {}) {
64
+ return withinBudget(ref.name, options, (signal) => fetchWithin(ref, options, signal));
65
+ }
66
+ /**
67
+ * Stages a companion set under its target names in one directory, the layout a
68
+ * downloaded set has, for engines that find their companions beside the
69
+ * checkpoint. The primary decides the outcome; a member the registry holds no
70
+ * description for is skipped.
71
+ */
72
+ export async function fetchFitStubSet(set, options = {}) {
73
+ return withinBudget(set.primary.name, options, (signal) => fetchSetWithin(set, options, signal));
74
+ }
75
+ function withinBudget(name, options, work) {
76
+ const budgetMs = options.budgetMs ?? FIT_STUB_BUDGET_MS;
77
+ const controller = new AbortController();
78
+ const onAbort = () => controller.abort(new Error('fit stub fetch aborted by the caller'));
79
+ options.signal?.addEventListener('abort', onAbort, { once: true });
80
+ let timer;
81
+ const expiry = new Promise((resolve) => {
82
+ timer = setTimeout(() => {
83
+ controller.abort(new Error('fit stub budget expired'));
84
+ resolve(unavailable('timed-out', `no fit stub for ${name} from the registry within ${budgetMs}ms`));
85
+ }, budgetMs);
86
+ });
87
+ return Promise.race([work(controller.signal), expiry]).finally(() => {
88
+ clearTimeout(timer);
89
+ options.signal?.removeEventListener('abort', onAbort);
90
+ });
91
+ }
92
+ // One directory per fetch, so two assessments of the same model never share a
93
+ // file that one of them is about to remove.
94
+ async function makeStagingDir(options) {
95
+ const root = options.cacheDir ?? getCacheDir('fit-stubs');
96
+ await fsPromises.mkdir(root, { recursive: true });
97
+ return fsPromises.mkdtemp(path.join(root, 'stub-'));
98
+ }
99
+ async function bindingFor(ref, options) {
100
+ const { registryPath, registrySource } = ref;
101
+ if (!registryPath || !registrySource)
102
+ return 'unresolvable-ref';
103
+ const getEntry = options.getEntry ?? defaultGetEntry;
104
+ const entry = await getEntry(registryPath, registrySource);
105
+ if (entry === null || entry === undefined)
106
+ return 'not-in-registry';
107
+ const binding = entry.fitBlobBinding;
108
+ if (binding === undefined || binding === null)
109
+ return 'no-fit-blob';
110
+ return binding;
111
+ }
112
+ async function stageOne(ref, binding, dest, options, signal) {
113
+ const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
114
+ let verified = false;
115
+ try {
116
+ await downloadBlob(binding, dest, signal);
117
+ const bytes = fs.statSync(dest).size;
118
+ if (bytes !== binding.byteLength) {
119
+ return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
120
+ }
121
+ // The record binds the description by digest; a stub that reads back
122
+ // differently is not the description the fitter should answer for.
123
+ const digest = await calculateFileChecksum(dest);
124
+ if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
125
+ return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
126
+ }
127
+ // Past the budget nobody is waiting for this result, so the stub must
128
+ // not be marked as kept.
129
+ if (signal.aborted)
130
+ return unavailable('timed-out');
131
+ verified = true;
132
+ return { status: 'staged', bytes };
133
+ }
134
+ catch (error) {
135
+ return unavailable('download-failed', describe(error));
136
+ }
137
+ finally {
138
+ // A short, corrupt, half-written or late stub must not outlive the call.
139
+ if (!verified)
140
+ await fsPromises.rm(dest, { force: true }).catch(() => { });
141
+ }
142
+ }
143
+ async function fetchWithin(ref, options, signal) {
144
+ const logger = options.logger ?? getEngineLogger();
145
+ try {
146
+ const binding = await bindingFor(ref, options);
147
+ if (typeof binding === 'string')
148
+ return unavailable(binding);
149
+ if (signal.aborted)
150
+ return unavailable('timed-out');
151
+ const dir = await makeStagingDir(options);
152
+ const dest = path.join(dir, `${binding.sha256}.gguf`);
153
+ const staged = await stageOne(ref, binding, dest, options, signal);
154
+ if (staged.status !== 'staged') {
155
+ await removeStub(dest);
156
+ return staged;
157
+ }
158
+ return { status: 'ready', path: dest, bytes: staged.bytes };
159
+ }
160
+ catch (error) {
161
+ logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
162
+ return unavailable('download-failed', describe(error));
163
+ }
164
+ }
165
+ async function fetchSetWithin(set, options, signal) {
166
+ const logger = options.logger ?? getEngineLogger();
167
+ let dir;
168
+ let ready = false;
169
+ try {
170
+ const binding = await bindingFor(set.primary, options);
171
+ if (typeof binding === 'string')
172
+ return unavailable(binding);
173
+ if (signal.aborted)
174
+ return unavailable('timed-out');
175
+ dir = await makeStagingDir(options);
176
+ const dest = path.join(dir, set.primary.targetName);
177
+ const staged = await stageOne(set.primary, binding, dest, options, signal);
178
+ if (staged.status !== 'staged')
179
+ return staged;
180
+ for (const member of set.others) {
181
+ if (signal.aborted)
182
+ break;
183
+ const memberBinding = await bindingFor(member, options);
184
+ if (typeof memberBinding === 'string') {
185
+ logger.debug(`companion ${member.targetName} has no fit stub: ${memberBinding}`);
186
+ continue;
187
+ }
188
+ const memberDest = path.join(dir, member.targetName);
189
+ const memberStaged = await stageOne(member, memberBinding, memberDest, options, signal);
190
+ if (memberStaged.status !== 'staged') {
191
+ logger.debug(`companion ${member.targetName} was not staged`);
192
+ }
193
+ }
194
+ // The budget races this work and has already answered the caller once it
195
+ // expires, so a set finished after that point is staged for nobody.
196
+ // The budget answers the caller on its own, so a set finished after it
197
+ // expired is staged for nobody.
198
+ if (signal.aborted)
199
+ return unavailable('timed-out');
200
+ ready = true;
201
+ return { status: 'ready', path: dest, bytes: staged.bytes };
202
+ }
203
+ catch (error) {
204
+ logger.debug(`fit stub set for ${set.primary.name} unavailable: ${describe(error)}`);
205
+ return unavailable('download-failed', describe(error));
206
+ }
207
+ finally {
208
+ if (!ready && dir !== undefined) {
209
+ await fsPromises.rm(dir, { recursive: true, force: true }).catch(() => { });
210
+ }
211
+ }
212
+ }
@@ -0,0 +1,27 @@
1
+ import { type LoadDescription } from './resolve-load-from-stubs.js';
2
+ import type { FitStubOptions, FitStubUnavailableReason } from './fetch-fit-stub.js';
3
+ import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
4
+ export type LoadFitOutcome = {
5
+ status: 'projected';
6
+ fit: AdvisoryFitOutcome;
7
+ } | {
8
+ status: 'no-stub';
9
+ reason: FitStubUnavailableReason;
10
+ message?: string;
11
+ } | {
12
+ status: 'unsupported-load';
13
+ detail: string;
14
+ };
15
+ export interface LoadFitOptions {
16
+ stub?: FitStubOptions;
17
+ fit?: AdvisoryFitOptions;
18
+ }
19
+ /**
20
+ * Projects the load a caller describes, before its weights exist locally.
21
+ *
22
+ * The same probe `loadModel` runs, over the same config and artifacts that
23
+ * load's own plugin resolves, pointed at files that are tens of KB instead of
24
+ * gigabytes. It inherits the probe's fail-open contract: every failure is an
25
+ * `unknown` verdict or a non-projected outcome, never a throw.
26
+ */
27
+ export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
@@ -0,0 +1,31 @@
1
+ import { resolveLoadFromStubs } from './resolve-load-from-stubs.js';
2
+ import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
3
+ /**
4
+ * Projects the load a caller describes, before its weights exist locally.
5
+ *
6
+ * The same probe `loadModel` runs, over the same config and artifacts that
7
+ * load's own plugin resolves, pointed at files that are tens of KB instead of
8
+ * gigabytes. It inherits the probe's fail-open contract: every failure is an
9
+ * `unknown` verdict or a non-projected outcome, never a throw.
10
+ */
11
+ export async function projectFitFromLoad(load, modelId, options = {}) {
12
+ const resolved = await resolveLoadFromStubs(load, options.stub);
13
+ if (resolved.status !== 'resolved')
14
+ return resolved;
15
+ try {
16
+ const fit = await runAdvisoryFitCheck({
17
+ modelId,
18
+ modelType: load.modelType,
19
+ modelPath: resolved.modelPath,
20
+ modelConfig: resolved.modelConfig,
21
+ artifacts: resolved.artifacts,
22
+ // Nothing here assembles a shard set, so no load may claim the fitter
23
+ // is looking at a split model.
24
+ isShardedModel: false
25
+ }, options.fit);
26
+ return { status: 'projected', fit };
27
+ }
28
+ finally {
29
+ await resolved.release();
30
+ }
31
+ }
@@ -0,0 +1,38 @@
1
+ import type { ModelSrcInput } from '../../../schemas/index.js';
2
+ import type { CanonicalModelType } from '../../../schemas/index.js';
3
+ import { type FitStubOptions, type FitStubUnavailableReason } from './fetch-fit-stub.js';
4
+ export interface LoadDescription {
5
+ /** Absent for the loads `loadModel` also takes without one. */
6
+ modelSrc?: ModelSrcInput | undefined;
7
+ modelType: CanonicalModelType;
8
+ modelConfig?: Record<string, unknown> | undefined;
9
+ }
10
+ export type ResolvedLoad = {
11
+ status: 'resolved';
12
+ modelPath: string;
13
+ modelConfig: unknown;
14
+ artifacts: Record<string, string>;
15
+ /** Removes every stub staged for this load. Never throws. */
16
+ release: () => Promise<void>;
17
+ };
18
+ export type LoadResolution = ResolvedLoad | {
19
+ status: 'no-stub';
20
+ reason: FitStubUnavailableReason;
21
+ message?: string;
22
+ } | {
23
+ status: 'unsupported-load';
24
+ detail: string;
25
+ };
26
+ /**
27
+ * Resolves a load the way `loadModel` would, against the registry's weightless
28
+ * descriptions instead of the weights.
29
+ *
30
+ * The plugin's own `resolveConfig` does the work: it strips the source fields
31
+ * from the config and returns the artifact paths keyed the way its engine
32
+ * expects. Only what a source resolves to differs here, so the config and the
33
+ * keys are the real load's.
34
+ *
35
+ * A source already on disk is read directly, since it describes the load more
36
+ * exactly than a stub does.
37
+ */
38
+ export declare function resolveLoadFromStubs(load: LoadDescription, options?: FitStubOptions): Promise<LoadResolution>;
@@ -0,0 +1,138 @@
1
+ import fs from 'bare-fs';
2
+ import { getPlugin } from '../../../plugins/registry.js';
3
+ import { getRuntimeContext } from '../../../runtime/state.js';
4
+ import { formatZodError } from '../../../utils/zod-error.js';
5
+ import { getModelByPath } from '../../../models/registry/index.js';
6
+ import { fetchFitStub, fetchFitStubSet, removeStub } from './fetch-fit-stub.js';
7
+ function srcString(modelSrc) {
8
+ return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
9
+ }
10
+ /**
11
+ * The stub ref a source carries, or `undefined` where it names no registry
12
+ * artifact. A catalog constant carries the checksum and coordinates; a bare
13
+ * string names a location and nothing the registry can be asked about.
14
+ */
15
+ function stubRefFor(modelSrc) {
16
+ if (typeof modelSrc === 'string')
17
+ return undefined;
18
+ const { sha256Checksum, registryPath, registrySource } = modelSrc;
19
+ if (sha256Checksum === undefined || registryPath === undefined)
20
+ return undefined;
21
+ return {
22
+ name: modelSrc.name ?? modelSrc.modelId ?? registryPath,
23
+ sha256Checksum,
24
+ registryPath,
25
+ ...(registrySource !== undefined && { registrySource })
26
+ };
27
+ }
28
+ function stubSetFor(registryPath) {
29
+ const companionSet = getModelByPath(registryPath)?.companionSet;
30
+ if (companionSet === undefined)
31
+ return undefined;
32
+ let primary;
33
+ const others = [];
34
+ for (const file of companionSet.files) {
35
+ const member = {
36
+ name: file.targetName,
37
+ sha256Checksum: file.sha256Checksum,
38
+ registryPath: file.registryPath,
39
+ registrySource: file.registrySource,
40
+ targetName: file.targetName
41
+ };
42
+ if (file.key === companionSet.primaryKey)
43
+ primary = member;
44
+ else
45
+ others.push(member);
46
+ }
47
+ return primary === undefined ? undefined : { primary, others };
48
+ }
49
+ class NoStubError extends Error {
50
+ reason;
51
+ constructor(reason, message) {
52
+ super(message ?? reason);
53
+ this.reason = reason;
54
+ }
55
+ }
56
+ /**
57
+ * Resolves a load the way `loadModel` would, against the registry's weightless
58
+ * descriptions instead of the weights.
59
+ *
60
+ * The plugin's own `resolveConfig` does the work: it strips the source fields
61
+ * from the config and returns the artifact paths keyed the way its engine
62
+ * expects. Only what a source resolves to differs here, so the config and the
63
+ * keys are the real load's.
64
+ *
65
+ * A source already on disk is read directly, since it describes the load more
66
+ * exactly than a stub does.
67
+ */
68
+ export async function resolveLoadFromStubs(load, options = {}) {
69
+ const plugin = getPlugin(load.modelType);
70
+ if (!plugin) {
71
+ return { status: 'unsupported-load', detail: `no plugin registered for ${load.modelType}` };
72
+ }
73
+ const parsed = plugin.loadConfigSchema.safeParse(load.modelConfig ?? {});
74
+ if (!parsed.success) {
75
+ return { status: 'unsupported-load', detail: formatZodError(parsed.error) };
76
+ }
77
+ const modelConfig = parsed.data;
78
+ const staged = [];
79
+ const release = async () => {
80
+ for (const path of staged)
81
+ await removeStub(path);
82
+ };
83
+ const resolveModelPath = async (src) => {
84
+ const location = srcString(src);
85
+ if (location !== '' && fs.existsSync(location))
86
+ return location;
87
+ const ref = stubRefFor(src);
88
+ if (ref === undefined)
89
+ throw new NoStubError('unresolvable-ref', location);
90
+ const set = stubSetFor(ref.registryPath);
91
+ const stub = set === undefined ? await fetchFitStub(ref, options) : await fetchFitStubSet(set, options);
92
+ if (stub.status !== 'ready')
93
+ throw new NoStubError(stub.reason, stub.message);
94
+ staged.push(stub.path);
95
+ return stub.path;
96
+ };
97
+ try {
98
+ const modelPath = load.modelSrc === undefined ? '' : await resolveModelPath(load.modelSrc);
99
+ if (plugin.resolveConfig === undefined) {
100
+ return {
101
+ status: 'resolved',
102
+ modelPath,
103
+ modelConfig,
104
+ artifacts: {},
105
+ release
106
+ };
107
+ }
108
+ const context = {
109
+ resolveModelPath,
110
+ modelSrc: load.modelSrc === undefined ? '' : srcString(load.modelSrc),
111
+ modelType: load.modelType,
112
+ ...(getRuntimeContext().platform !== undefined && {
113
+ platform: getRuntimeContext().platform
114
+ })
115
+ };
116
+ const resolved = await plugin.resolveConfig(modelConfig, context);
117
+ return {
118
+ status: 'resolved',
119
+ modelPath,
120
+ modelConfig: resolved.config,
121
+ artifacts: (resolved.artifacts ?? {}),
122
+ release
123
+ };
124
+ }
125
+ catch (error) {
126
+ await release();
127
+ if (error instanceof NoStubError) {
128
+ return error.message === error.reason
129
+ ? { status: 'no-stub', reason: error.reason }
130
+ : { status: 'no-stub', reason: error.reason, message: error.message };
131
+ }
132
+ // A config the plugin refuses describes a load that would not start either.
133
+ return {
134
+ status: 'unsupported-load',
135
+ detail: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
136
+ };
137
+ }
138
+ }
@@ -47,7 +47,7 @@ export function computeFloor(input) {
47
47
  reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
48
48
  return { bytes: artifactBytes, reasons, assumptions };
49
49
  }
50
- let contextTokens = workload.contextTokens;
50
+ let contextTokens = workload.contextTokens ?? facts.contextLength;
51
51
  if (contextTokens > facts.contextLength) {
52
52
  assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
53
53
  contextTokens = facts.contextLength;
@@ -59,7 +59,7 @@ export function computeFloor(input) {
59
59
  const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
60
60
  assumptions.push(element < 2
61
61
  ? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
62
- : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not expressible in a workload and would change these numbers');
62
+ : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
63
63
  reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
64
64
  return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
65
65
  }
@@ -1,12 +1,12 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
2
  import type { Logger } from '../../../logging/types.js';
3
3
  import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
4
- import type { runIsolatedFit } from './run-isolated-fit.js';
4
+ import { runFit as runFitDefault } from './run-fit.js';
5
5
  /**
6
6
  * `fit` and `does-not-fit` are projections of the load the SDK is about to run,
7
- * not admission decisions. `@qvac/model-fit` duplicates the loader's policy for
8
- * this experiment and the real loader neither consumes nor verifies the fitted
9
- * plan, so neither verdict is denial-grade and no verdict changes the load.
7
+ * not admission decisions. The real loader neither consumes nor verifies the
8
+ * fitted plan, so neither verdict is denial-grade and no verdict changes the
9
+ * load.
10
10
  *
11
11
  * The outcome is the wire shape: `loadModel` hands it to the model registry and
12
12
  * `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
@@ -31,12 +31,15 @@ export interface AdvisoryFitOptions {
31
31
  enabled?: boolean;
32
32
  mobile?: boolean;
33
33
  timeoutMs?: number;
34
- runFit?: typeof runIsolatedFit;
34
+ runFit?: typeof runFitDefault;
35
35
  logger?: Logger;
36
+ availableSystemBytes?: () => Promise<number | undefined>;
37
+ countsDeviceRows?: () => Promise<boolean>;
36
38
  residentModelBytes?: () => Promise<number>;
37
39
  }
38
40
  /**
39
- * Runs the advisory llama.cpp fit check for a load that is about to start.
41
+ * Runs the advisory fit check for a load that is about to start, against the
42
+ * fitter belonging to the engine that will run it.
40
43
  *
41
44
  * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
42
45
  * malformed response, and an unexpected internal error all resolve to `unknown`