@qvac/inference 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +90 -24
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
|
@@ -12,8 +12,13 @@ import { getEngineLogger } from '../../../logging/index.js';
|
|
|
12
12
|
* capped. Work still in flight at expiry is abandoned: the download is
|
|
13
13
|
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
14
14
|
* nothing waiting for it.
|
|
15
|
+
*
|
|
16
|
+
* One budget covers a whole companion set, so it holds the slowest case: every
|
|
17
|
+
* member of a multi-stage voice model staged over a mobile network. It sits
|
|
18
|
+
* above the client's per-attempt timeout rather than on it, so an attempt that
|
|
19
|
+
* is about to answer is not cut at the moment it would have.
|
|
15
20
|
*/
|
|
16
|
-
export const FIT_STUB_BUDGET_MS =
|
|
21
|
+
export const FIT_STUB_BUDGET_MS = 40_000;
|
|
17
22
|
async function defaultGetEntry(registryPath, registrySource) {
|
|
18
23
|
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
19
24
|
const client = await getRegistryClient();
|
|
@@ -56,6 +61,18 @@ function describe(error) {
|
|
|
56
61
|
* llama's shard naming to read any of it — assembling that is not done here.
|
|
57
62
|
*/
|
|
58
63
|
export async function fetchFitStub(ref, options = {}) {
|
|
64
|
+
return withinBudget(ref.name, options, (signal) => fetchWithin(ref, options, signal));
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Stages a companion set under its target names in one directory, the layout a
|
|
68
|
+
* downloaded set has, for engines that find their companions beside the
|
|
69
|
+
* checkpoint. The primary decides the outcome; a member the registry holds no
|
|
70
|
+
* description for is skipped.
|
|
71
|
+
*/
|
|
72
|
+
export async function fetchFitStubSet(set, options = {}) {
|
|
73
|
+
return withinBudget(set.primary.name, options, (signal) => fetchSetWithin(set, options, signal));
|
|
74
|
+
}
|
|
75
|
+
function withinBudget(name, options, work) {
|
|
59
76
|
const budgetMs = options.budgetMs ?? FIT_STUB_BUDGET_MS;
|
|
60
77
|
const controller = new AbortController();
|
|
61
78
|
const onAbort = () => controller.abort(new Error('fit stub fetch aborted by the caller'));
|
|
@@ -64,70 +81,132 @@ export async function fetchFitStub(ref, options = {}) {
|
|
|
64
81
|
const expiry = new Promise((resolve) => {
|
|
65
82
|
timer = setTimeout(() => {
|
|
66
83
|
controller.abort(new Error('fit stub budget expired'));
|
|
67
|
-
resolve(unavailable('timed-out', `no fit stub for ${
|
|
84
|
+
resolve(unavailable('timed-out', `no fit stub for ${name} from the registry within ${budgetMs}ms`));
|
|
68
85
|
}, budgetMs);
|
|
69
86
|
});
|
|
87
|
+
return Promise.race([work(controller.signal), expiry]).finally(() => {
|
|
88
|
+
clearTimeout(timer);
|
|
89
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
// One directory per fetch, so two assessments of the same model never share a
|
|
93
|
+
// file that one of them is about to remove.
|
|
94
|
+
async function makeStagingDir(options) {
|
|
95
|
+
const root = options.cacheDir ?? getCacheDir('fit-stubs');
|
|
96
|
+
await fsPromises.mkdir(root, { recursive: true });
|
|
97
|
+
return fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
98
|
+
}
|
|
99
|
+
async function bindingFor(ref, options) {
|
|
100
|
+
const { registryPath, registrySource } = ref;
|
|
101
|
+
if (!registryPath || !registrySource)
|
|
102
|
+
return 'unresolvable-ref';
|
|
103
|
+
const getEntry = options.getEntry ?? defaultGetEntry;
|
|
104
|
+
const entry = await getEntry(registryPath, registrySource);
|
|
105
|
+
if (entry === null || entry === undefined)
|
|
106
|
+
return 'not-in-registry';
|
|
107
|
+
const binding = entry.fitBlobBinding;
|
|
108
|
+
if (binding === undefined || binding === null)
|
|
109
|
+
return 'no-fit-blob';
|
|
110
|
+
return binding;
|
|
111
|
+
}
|
|
112
|
+
async function stageOne(ref, binding, dest, options, signal) {
|
|
113
|
+
const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
|
|
114
|
+
let verified = false;
|
|
70
115
|
try {
|
|
71
|
-
|
|
116
|
+
await downloadBlob(binding, dest, signal);
|
|
117
|
+
const bytes = fs.statSync(dest).size;
|
|
118
|
+
if (bytes !== binding.byteLength) {
|
|
119
|
+
return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
|
|
120
|
+
}
|
|
121
|
+
// The record binds the description by digest; a stub that reads back
|
|
122
|
+
// differently is not the description the fitter should answer for.
|
|
123
|
+
const digest = await calculateFileChecksum(dest);
|
|
124
|
+
if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
|
|
125
|
+
return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
|
|
126
|
+
}
|
|
127
|
+
// Past the budget nobody is waiting for this result, so the stub must
|
|
128
|
+
// not be marked as kept.
|
|
129
|
+
if (signal.aborted)
|
|
130
|
+
return unavailable('timed-out');
|
|
131
|
+
verified = true;
|
|
132
|
+
return { status: 'staged', bytes };
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
return unavailable('download-failed', describe(error));
|
|
72
136
|
}
|
|
73
137
|
finally {
|
|
74
|
-
|
|
75
|
-
|
|
138
|
+
// A short, corrupt, half-written or late stub must not outlive the call.
|
|
139
|
+
if (!verified)
|
|
140
|
+
await fsPromises.rm(dest, { force: true }).catch(() => { });
|
|
76
141
|
}
|
|
77
142
|
}
|
|
78
143
|
async function fetchWithin(ref, options, signal) {
|
|
79
144
|
const logger = options.logger ?? getEngineLogger();
|
|
80
145
|
try {
|
|
81
|
-
const
|
|
82
|
-
if (
|
|
83
|
-
return unavailable(
|
|
84
|
-
const getEntry = options.getEntry ?? defaultGetEntry;
|
|
85
|
-
const entry = await getEntry(registryPath, registrySource);
|
|
86
|
-
if (entry === null || entry === undefined)
|
|
87
|
-
return unavailable('not-in-registry');
|
|
88
|
-
const binding = entry.fitBlobBinding;
|
|
89
|
-
if (binding === undefined || binding === null)
|
|
90
|
-
return unavailable('no-fit-blob');
|
|
146
|
+
const binding = await bindingFor(ref, options);
|
|
147
|
+
if (typeof binding === 'string')
|
|
148
|
+
return unavailable(binding);
|
|
91
149
|
if (signal.aborted)
|
|
92
150
|
return unavailable('timed-out');
|
|
93
|
-
|
|
94
|
-
// a file that one of them is about to remove.
|
|
95
|
-
const root = options.cacheDir ?? getCacheDir('fit-stubs');
|
|
96
|
-
await fsPromises.mkdir(root, { recursive: true });
|
|
97
|
-
const dir = await fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
151
|
+
const dir = await makeStagingDir(options);
|
|
98
152
|
const dest = path.join(dir, `${binding.sha256}.gguf`);
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
153
|
+
const staged = await stageOne(ref, binding, dest, options, signal);
|
|
154
|
+
if (staged.status !== 'staged') {
|
|
155
|
+
await removeStub(dest);
|
|
156
|
+
return staged;
|
|
157
|
+
}
|
|
158
|
+
return { status: 'ready', path: dest, bytes: staged.bytes };
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
|
|
162
|
+
return unavailable('download-failed', describe(error));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async function fetchSetWithin(set, options, signal) {
|
|
166
|
+
const logger = options.logger ?? getEngineLogger();
|
|
167
|
+
let dir;
|
|
168
|
+
let ready = false;
|
|
169
|
+
try {
|
|
170
|
+
const binding = await bindingFor(set.primary, options);
|
|
171
|
+
if (typeof binding === 'string')
|
|
172
|
+
return unavailable(binding);
|
|
173
|
+
if (signal.aborted)
|
|
174
|
+
return unavailable('timed-out');
|
|
175
|
+
dir = await makeStagingDir(options);
|
|
176
|
+
const dest = path.join(dir, set.primary.targetName);
|
|
177
|
+
const staged = await stageOne(set.primary, binding, dest, options, signal);
|
|
178
|
+
if (staged.status !== 'staged')
|
|
179
|
+
return staged;
|
|
180
|
+
for (const member of set.others) {
|
|
181
|
+
if (signal.aborted)
|
|
182
|
+
break;
|
|
183
|
+
const memberBinding = await bindingFor(member, options);
|
|
184
|
+
if (typeof memberBinding === 'string') {
|
|
185
|
+
logger.debug(`companion ${member.targetName} has no fit stub: ${memberBinding}`);
|
|
186
|
+
continue;
|
|
106
187
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
|
|
188
|
+
const memberDest = path.join(dir, member.targetName);
|
|
189
|
+
const memberStaged = await stageOne(member, memberBinding, memberDest, options, signal);
|
|
190
|
+
if (memberStaged.status !== 'staged') {
|
|
191
|
+
logger.debug(`companion ${member.targetName} was not staged`);
|
|
112
192
|
}
|
|
113
|
-
// Past the budget nobody is waiting for this result, so the stub must
|
|
114
|
-
// not be marked as kept.
|
|
115
|
-
if (signal.aborted)
|
|
116
|
-
return unavailable('timed-out');
|
|
117
|
-
verified = true;
|
|
118
|
-
return { status: 'ready', path: dest, bytes };
|
|
119
|
-
}
|
|
120
|
-
catch (error) {
|
|
121
|
-
return unavailable('download-failed', describe(error));
|
|
122
|
-
}
|
|
123
|
-
finally {
|
|
124
|
-
// A short, corrupt, half-written or late stub must not outlive the call.
|
|
125
|
-
if (!verified)
|
|
126
|
-
await removeStub(dest);
|
|
127
193
|
}
|
|
194
|
+
// The budget races this work and has already answered the caller once it
|
|
195
|
+
// expires, so a set finished after that point is staged for nobody.
|
|
196
|
+
// The budget answers the caller on its own, so a set finished after it
|
|
197
|
+
// expired is staged for nobody.
|
|
198
|
+
if (signal.aborted)
|
|
199
|
+
return unavailable('timed-out');
|
|
200
|
+
ready = true;
|
|
201
|
+
return { status: 'ready', path: dest, bytes: staged.bytes };
|
|
128
202
|
}
|
|
129
203
|
catch (error) {
|
|
130
|
-
logger.debug(`fit stub for ${
|
|
204
|
+
logger.debug(`fit stub set for ${set.primary.name} unavailable: ${describe(error)}`);
|
|
131
205
|
return unavailable('download-failed', describe(error));
|
|
132
206
|
}
|
|
207
|
+
finally {
|
|
208
|
+
if (!ready && dir !== undefined) {
|
|
209
|
+
await fsPromises.rm(dir, { recursive: true, force: true }).catch(() => { });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
133
212
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type LoadDescription } from './resolve-load-from-stubs.js';
|
|
2
|
+
import type { FitStubOptions, FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
3
|
+
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
|
|
4
|
+
export type LoadFitOutcome = {
|
|
5
|
+
status: 'projected';
|
|
6
|
+
fit: AdvisoryFitOutcome;
|
|
7
|
+
} | {
|
|
8
|
+
status: 'no-stub';
|
|
9
|
+
reason: FitStubUnavailableReason;
|
|
10
|
+
message?: string;
|
|
11
|
+
} | {
|
|
12
|
+
status: 'unsupported-load';
|
|
13
|
+
detail: string;
|
|
14
|
+
};
|
|
15
|
+
export interface LoadFitOptions {
|
|
16
|
+
stub?: FitStubOptions;
|
|
17
|
+
fit?: AdvisoryFitOptions;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Projects the load a caller describes, before its weights exist locally.
|
|
21
|
+
*
|
|
22
|
+
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
23
|
+
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
24
|
+
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
25
|
+
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
26
|
+
*/
|
|
27
|
+
export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { resolveLoadFromStubs } from './resolve-load-from-stubs.js';
|
|
2
|
+
import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
|
|
3
|
+
/**
|
|
4
|
+
* Projects the load a caller describes, before its weights exist locally.
|
|
5
|
+
*
|
|
6
|
+
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
7
|
+
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
8
|
+
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
9
|
+
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
10
|
+
*/
|
|
11
|
+
export async function projectFitFromLoad(load, modelId, options = {}) {
|
|
12
|
+
const resolved = await resolveLoadFromStubs(load, options.stub);
|
|
13
|
+
if (resolved.status !== 'resolved')
|
|
14
|
+
return resolved;
|
|
15
|
+
try {
|
|
16
|
+
const fit = await runAdvisoryFitCheck({
|
|
17
|
+
modelId,
|
|
18
|
+
modelType: load.modelType,
|
|
19
|
+
modelPath: resolved.modelPath,
|
|
20
|
+
modelConfig: resolved.modelConfig,
|
|
21
|
+
artifacts: resolved.artifacts,
|
|
22
|
+
// Nothing here assembles a shard set, so no load may claim the fitter
|
|
23
|
+
// is looking at a split model.
|
|
24
|
+
isShardedModel: false
|
|
25
|
+
}, options.fit);
|
|
26
|
+
return { status: 'projected', fit };
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
await resolved.release();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { ModelSrcInput } from '../../../schemas/index.js';
|
|
2
|
+
import type { CanonicalModelType } from '../../../schemas/index.js';
|
|
3
|
+
import { type FitStubOptions, type FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
4
|
+
export interface LoadDescription {
|
|
5
|
+
/** Absent for the loads `loadModel` also takes without one. */
|
|
6
|
+
modelSrc?: ModelSrcInput | undefined;
|
|
7
|
+
modelType: CanonicalModelType;
|
|
8
|
+
modelConfig?: Record<string, unknown> | undefined;
|
|
9
|
+
}
|
|
10
|
+
export type ResolvedLoad = {
|
|
11
|
+
status: 'resolved';
|
|
12
|
+
modelPath: string;
|
|
13
|
+
modelConfig: unknown;
|
|
14
|
+
artifacts: Record<string, string>;
|
|
15
|
+
/** Removes every stub staged for this load. Never throws. */
|
|
16
|
+
release: () => Promise<void>;
|
|
17
|
+
};
|
|
18
|
+
export type LoadResolution = ResolvedLoad | {
|
|
19
|
+
status: 'no-stub';
|
|
20
|
+
reason: FitStubUnavailableReason;
|
|
21
|
+
message?: string;
|
|
22
|
+
} | {
|
|
23
|
+
status: 'unsupported-load';
|
|
24
|
+
detail: string;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Resolves a load the way `loadModel` would, against the registry's weightless
|
|
28
|
+
* descriptions instead of the weights.
|
|
29
|
+
*
|
|
30
|
+
* The plugin's own `resolveConfig` does the work: it strips the source fields
|
|
31
|
+
* from the config and returns the artifact paths keyed the way its engine
|
|
32
|
+
* expects. Only what a source resolves to differs here, so the config and the
|
|
33
|
+
* keys are the real load's.
|
|
34
|
+
*
|
|
35
|
+
* A source already on disk is read directly, since it describes the load more
|
|
36
|
+
* exactly than a stub does.
|
|
37
|
+
*/
|
|
38
|
+
export declare function resolveLoadFromStubs(load: LoadDescription, options?: FitStubOptions): Promise<LoadResolution>;
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import fs from 'bare-fs';
|
|
2
|
+
import { getPlugin } from '../../../plugins/registry.js';
|
|
3
|
+
import { getRuntimeContext } from '../../../runtime/state.js';
|
|
4
|
+
import { formatZodError } from '../../../utils/zod-error.js';
|
|
5
|
+
import { getModelByPath } from '../../../models/registry/index.js';
|
|
6
|
+
import { fetchFitStub, fetchFitStubSet, removeStub } from './fetch-fit-stub.js';
|
|
7
|
+
function srcString(modelSrc) {
|
|
8
|
+
return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* The stub ref a source carries, or `undefined` where it names no registry
|
|
12
|
+
* artifact. A catalog constant carries the checksum and coordinates; a bare
|
|
13
|
+
* string names a location and nothing the registry can be asked about.
|
|
14
|
+
*/
|
|
15
|
+
function stubRefFor(modelSrc) {
|
|
16
|
+
if (typeof modelSrc === 'string')
|
|
17
|
+
return undefined;
|
|
18
|
+
const { sha256Checksum, registryPath, registrySource } = modelSrc;
|
|
19
|
+
if (sha256Checksum === undefined || registryPath === undefined)
|
|
20
|
+
return undefined;
|
|
21
|
+
return {
|
|
22
|
+
name: modelSrc.name ?? modelSrc.modelId ?? registryPath,
|
|
23
|
+
sha256Checksum,
|
|
24
|
+
registryPath,
|
|
25
|
+
...(registrySource !== undefined && { registrySource })
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function stubSetFor(registryPath) {
|
|
29
|
+
const companionSet = getModelByPath(registryPath)?.companionSet;
|
|
30
|
+
if (companionSet === undefined)
|
|
31
|
+
return undefined;
|
|
32
|
+
let primary;
|
|
33
|
+
const others = [];
|
|
34
|
+
for (const file of companionSet.files) {
|
|
35
|
+
const member = {
|
|
36
|
+
name: file.targetName,
|
|
37
|
+
sha256Checksum: file.sha256Checksum,
|
|
38
|
+
registryPath: file.registryPath,
|
|
39
|
+
registrySource: file.registrySource,
|
|
40
|
+
targetName: file.targetName
|
|
41
|
+
};
|
|
42
|
+
if (file.key === companionSet.primaryKey)
|
|
43
|
+
primary = member;
|
|
44
|
+
else
|
|
45
|
+
others.push(member);
|
|
46
|
+
}
|
|
47
|
+
return primary === undefined ? undefined : { primary, others };
|
|
48
|
+
}
|
|
49
|
+
class NoStubError extends Error {
|
|
50
|
+
reason;
|
|
51
|
+
constructor(reason, message) {
|
|
52
|
+
super(message ?? reason);
|
|
53
|
+
this.reason = reason;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Resolves a load the way `loadModel` would, against the registry's weightless
|
|
58
|
+
* descriptions instead of the weights.
|
|
59
|
+
*
|
|
60
|
+
* The plugin's own `resolveConfig` does the work: it strips the source fields
|
|
61
|
+
* from the config and returns the artifact paths keyed the way its engine
|
|
62
|
+
* expects. Only what a source resolves to differs here, so the config and the
|
|
63
|
+
* keys are the real load's.
|
|
64
|
+
*
|
|
65
|
+
* A source already on disk is read directly, since it describes the load more
|
|
66
|
+
* exactly than a stub does.
|
|
67
|
+
*/
|
|
68
|
+
export async function resolveLoadFromStubs(load, options = {}) {
|
|
69
|
+
const plugin = getPlugin(load.modelType);
|
|
70
|
+
if (!plugin) {
|
|
71
|
+
return { status: 'unsupported-load', detail: `no plugin registered for ${load.modelType}` };
|
|
72
|
+
}
|
|
73
|
+
const parsed = plugin.loadConfigSchema.safeParse(load.modelConfig ?? {});
|
|
74
|
+
if (!parsed.success) {
|
|
75
|
+
return { status: 'unsupported-load', detail: formatZodError(parsed.error) };
|
|
76
|
+
}
|
|
77
|
+
const modelConfig = parsed.data;
|
|
78
|
+
const staged = [];
|
|
79
|
+
const release = async () => {
|
|
80
|
+
for (const path of staged)
|
|
81
|
+
await removeStub(path);
|
|
82
|
+
};
|
|
83
|
+
const resolveModelPath = async (src) => {
|
|
84
|
+
const location = srcString(src);
|
|
85
|
+
if (location !== '' && fs.existsSync(location))
|
|
86
|
+
return location;
|
|
87
|
+
const ref = stubRefFor(src);
|
|
88
|
+
if (ref === undefined)
|
|
89
|
+
throw new NoStubError('unresolvable-ref', location);
|
|
90
|
+
const set = stubSetFor(ref.registryPath);
|
|
91
|
+
const stub = set === undefined ? await fetchFitStub(ref, options) : await fetchFitStubSet(set, options);
|
|
92
|
+
if (stub.status !== 'ready')
|
|
93
|
+
throw new NoStubError(stub.reason, stub.message);
|
|
94
|
+
staged.push(stub.path);
|
|
95
|
+
return stub.path;
|
|
96
|
+
};
|
|
97
|
+
try {
|
|
98
|
+
const modelPath = load.modelSrc === undefined ? '' : await resolveModelPath(load.modelSrc);
|
|
99
|
+
if (plugin.resolveConfig === undefined) {
|
|
100
|
+
return {
|
|
101
|
+
status: 'resolved',
|
|
102
|
+
modelPath,
|
|
103
|
+
modelConfig,
|
|
104
|
+
artifacts: {},
|
|
105
|
+
release
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
const context = {
|
|
109
|
+
resolveModelPath,
|
|
110
|
+
modelSrc: load.modelSrc === undefined ? '' : srcString(load.modelSrc),
|
|
111
|
+
modelType: load.modelType,
|
|
112
|
+
...(getRuntimeContext().platform !== undefined && {
|
|
113
|
+
platform: getRuntimeContext().platform
|
|
114
|
+
})
|
|
115
|
+
};
|
|
116
|
+
const resolved = await plugin.resolveConfig(modelConfig, context);
|
|
117
|
+
return {
|
|
118
|
+
status: 'resolved',
|
|
119
|
+
modelPath,
|
|
120
|
+
modelConfig: resolved.config,
|
|
121
|
+
artifacts: (resolved.artifacts ?? {}),
|
|
122
|
+
release
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
await release();
|
|
127
|
+
if (error instanceof NoStubError) {
|
|
128
|
+
return error.message === error.reason
|
|
129
|
+
? { status: 'no-stub', reason: error.reason }
|
|
130
|
+
: { status: 'no-stub', reason: error.reason, message: error.message };
|
|
131
|
+
}
|
|
132
|
+
// A config the plugin refuses describes a load that would not start either.
|
|
133
|
+
return {
|
|
134
|
+
status: 'unsupported-load',
|
|
135
|
+
detail: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
@@ -47,7 +47,7 @@ export function computeFloor(input) {
|
|
|
47
47
|
reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
|
|
48
48
|
return { bytes: artifactBytes, reasons, assumptions };
|
|
49
49
|
}
|
|
50
|
-
let contextTokens = workload.contextTokens;
|
|
50
|
+
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
51
51
|
if (contextTokens > facts.contextLength) {
|
|
52
52
|
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
53
53
|
contextTokens = facts.contextLength;
|
|
@@ -59,7 +59,7 @@ export function computeFloor(input) {
|
|
|
59
59
|
const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
|
|
60
60
|
assumptions.push(element < 2
|
|
61
61
|
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
|
-
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not
|
|
62
|
+
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
63
63
|
reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
|
|
64
64
|
return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
|
|
65
65
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { Logger } from '../../../logging/types.js';
|
|
3
3
|
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
|
|
4
|
-
import
|
|
4
|
+
import { runFit as runFitDefault } from './run-fit.js';
|
|
5
5
|
/**
|
|
6
6
|
* `fit` and `does-not-fit` are projections of the load the SDK is about to run,
|
|
7
|
-
* not admission decisions.
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* not admission decisions. The real loader neither consumes nor verifies the
|
|
8
|
+
* fitted plan, so neither verdict is denial-grade and no verdict changes the
|
|
9
|
+
* load.
|
|
10
10
|
*
|
|
11
11
|
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
12
12
|
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
@@ -31,12 +31,15 @@ export interface AdvisoryFitOptions {
|
|
|
31
31
|
enabled?: boolean;
|
|
32
32
|
mobile?: boolean;
|
|
33
33
|
timeoutMs?: number;
|
|
34
|
-
runFit?: typeof
|
|
34
|
+
runFit?: typeof runFitDefault;
|
|
35
35
|
logger?: Logger;
|
|
36
|
+
availableSystemBytes?: () => Promise<number | undefined>;
|
|
37
|
+
countsDeviceRows?: () => Promise<boolean>;
|
|
36
38
|
residentModelBytes?: () => Promise<number>;
|
|
37
39
|
}
|
|
38
40
|
/**
|
|
39
|
-
* Runs the advisory
|
|
41
|
+
* Runs the advisory fit check for a load that is about to start, against the
|
|
42
|
+
* fitter belonging to the engine that will run it.
|
|
40
43
|
*
|
|
41
44
|
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
42
45
|
* malformed response, and an unexpected internal error all resolve to `unknown`
|