@qvac/inference 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +57 -17
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +67 -0
- package/dist/schemas/assess-model-fit.js +56 -2
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +816 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +186 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +88 -8
- package/dist/schemas/llamacpp-config.js +108 -8
- package/dist/schemas/load-model.d.ts +2906 -206
- package/dist/schemas/model-config-schemas.d.ts +251 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
package/dist/plugins/registry.js
CHANGED
|
@@ -5,9 +5,41 @@ import { createAddonLoggerCallback } from '../logging/addon.js';
|
|
|
5
5
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
6
|
import { formatZodError } from '../utils/zod-error.js';
|
|
7
7
|
const plugins = new Map();
|
|
8
|
+
/** Addon logging modules wired through a resolver, keyed by namespace. */
|
|
9
|
+
const lazyAddonLoggers = new Map();
|
|
10
|
+
/** In-flight resolutions, so concurrent loads wire a namespace exactly once. */
|
|
11
|
+
const pendingAddonLoggers = new Map();
|
|
12
|
+
function getLoggingResolver(plugin) {
|
|
13
|
+
const declared = plugin.logging?.module;
|
|
14
|
+
return typeof declared === 'function' ? declared : undefined;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The module a plugin supplied directly, or `undefined` when it supplied a
|
|
18
|
+
* resolver. Resolver-based plugins are therefore invisible to
|
|
19
|
+
* registration-time shape validation, to the shared-module dedupe, and to the
|
|
20
|
+
* eager release sweeps; `ensureAddonLoggerReady` and `lazyAddonLoggers` cover
|
|
21
|
+
* them instead, keyed by namespace rather than held on the plugin.
|
|
22
|
+
*/
|
|
8
23
|
function getLoggingModule(plugin) {
|
|
24
|
+
if (getLoggingResolver(plugin))
|
|
25
|
+
return undefined;
|
|
9
26
|
return plugin.logging?.module;
|
|
10
27
|
}
|
|
28
|
+
/** CommonJS addons resolve through `import()` as a namespace with `default`. */
|
|
29
|
+
function unwrapLoggingModule(resolved) {
|
|
30
|
+
if (resolved && typeof resolved === 'object' && 'default' in resolved) {
|
|
31
|
+
const inner = resolved.default;
|
|
32
|
+
if (inner && typeof inner.setLogger === 'function')
|
|
33
|
+
return inner;
|
|
34
|
+
}
|
|
35
|
+
return resolved;
|
|
36
|
+
}
|
|
37
|
+
function assertLoggingModuleShape(modelType, candidate) {
|
|
38
|
+
if (!candidate || typeof candidate['setLogger'] !== 'function') {
|
|
39
|
+
throw new PluginLoggingInvalidError(modelType, 'logging.module must have a setLogger(callback) function');
|
|
40
|
+
}
|
|
41
|
+
return candidate;
|
|
42
|
+
}
|
|
11
43
|
function findPluginUsingLoggingModule(loggingModule) {
|
|
12
44
|
return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
|
|
13
45
|
}
|
|
@@ -42,12 +74,11 @@ export function registerPlugin(plugin) {
|
|
|
42
74
|
if (pluginProvidingTurboVecIndex) {
|
|
43
75
|
throw new PluginDefinitionInvalidError(plugin.modelType, `plugin "${plugin.modelType}" cannot provide turbovecIndexProvider because plugin "${pluginProvidingTurboVecIndex.modelType}" already provides it`);
|
|
44
76
|
}
|
|
45
|
-
// Validate logging module shape if provided
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
77
|
+
// Validate logging module shape if provided. A resolver is checked when it
|
|
78
|
+
// runs instead: calling it here to inspect its result would load the addon,
|
|
79
|
+
// which is the whole thing registration is meant to avoid.
|
|
80
|
+
if (plugin.logging?.module && !getLoggingResolver(plugin)) {
|
|
81
|
+
assertLoggingModuleShape(plugin.modelType, plugin.logging.module);
|
|
51
82
|
}
|
|
52
83
|
const loggingModule = getLoggingModule(plugin);
|
|
53
84
|
const pluginUsingLoggingModule = loggingModule
|
|
@@ -67,6 +98,91 @@ export function registerPlugins(pluginList) {
|
|
|
67
98
|
registerPlugin(plugin);
|
|
68
99
|
}
|
|
69
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Wires the addon logger of a plugin that supplied a resolver, loading the
|
|
103
|
+
* addon on the way. Call it once a model of this type is actually being
|
|
104
|
+
* created: it is the point where the addon is needed anyway, so a host
|
|
105
|
+
* missing that addon's platform package fails the load it asked for instead
|
|
106
|
+
* of failing every plugin registration at startup.
|
|
107
|
+
*
|
|
108
|
+
* A no-op for plugins that passed their module directly — those are already
|
|
109
|
+
* wired by `registerPlugin` — and for a namespace that is already wired,
|
|
110
|
+
* which is how two plugins over one addon (whisper and Parakeet over ASR)
|
|
111
|
+
* share a single `setLogger` call. Also a no-op when the registry was cleared
|
|
112
|
+
* while the addon was loading, so a shutdown that races a load leaves no
|
|
113
|
+
* logger behind.
|
|
114
|
+
*/
|
|
115
|
+
export async function ensureAddonLoggerReady(plugin) {
|
|
116
|
+
const resolver = getLoggingResolver(plugin);
|
|
117
|
+
const namespace = plugin.logging?.namespace;
|
|
118
|
+
if (!resolver || !namespace)
|
|
119
|
+
return;
|
|
120
|
+
if (lazyAddonLoggers.has(namespace))
|
|
121
|
+
return;
|
|
122
|
+
const pending = pendingAddonLoggers.get(namespace);
|
|
123
|
+
if (pending)
|
|
124
|
+
return pending;
|
|
125
|
+
const wiring = (async () => {
|
|
126
|
+
const resolved = unwrapLoggingModule(await resolver());
|
|
127
|
+
// Loading the addon takes real time, and the registry can be torn down
|
|
128
|
+
// while this is parked: `runCleanup` calls `clearRegistries()`
|
|
129
|
+
// synchronously and only then awaits `unloadAllModels()`, so a sweep can
|
|
130
|
+
// pass between the call above and this line. Wiring now would hand the
|
|
131
|
+
// addon a callback nothing will release — on Expo, a `js_ref_t` leaked
|
|
132
|
+
// into a dying isolate, which the next worklet's first `setLogger` trips
|
|
133
|
+
// over. Nothing has been wired yet, so dropping the resolved module is
|
|
134
|
+
// the whole cleanup.
|
|
135
|
+
if (!isNamespaceClaimed(namespace))
|
|
136
|
+
return;
|
|
137
|
+
const loggingModule = assertLoggingModuleShape(plugin.modelType, resolved);
|
|
138
|
+
for (const [wiredNamespace, wiredModule] of lazyAddonLoggers) {
|
|
139
|
+
if (wiredModule === loggingModule && wiredNamespace !== namespace) {
|
|
140
|
+
throw new PluginLoggingInvalidError(plugin.modelType, 'plugins sharing logging.module must use the same namespace');
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
loggingModule.setLogger(createAddonLoggerCallback(namespace));
|
|
144
|
+
lazyAddonLoggers.set(namespace, loggingModule);
|
|
145
|
+
})();
|
|
146
|
+
pendingAddonLoggers.set(namespace, wiring);
|
|
147
|
+
try {
|
|
148
|
+
await wiring;
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
pendingAddonLoggers.delete(namespace);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** Whether a registered resolver-based plugin still claims this namespace. */
|
|
155
|
+
function isNamespaceClaimed(namespace) {
|
|
156
|
+
return Array.from(plugins.values()).some((candidate) => getLoggingResolver(candidate) && candidate.logging?.namespace === namespace);
|
|
157
|
+
}
|
|
158
|
+
/** Releases a lazily-wired addon logger once no registered plugin claims it. */
|
|
159
|
+
function releaseLazyAddonLogger(namespace, modelType) {
|
|
160
|
+
if (namespace === undefined)
|
|
161
|
+
return;
|
|
162
|
+
const loggingModule = lazyAddonLoggers.get(namespace);
|
|
163
|
+
if (!loggingModule)
|
|
164
|
+
return;
|
|
165
|
+
if (isNamespaceClaimed(namespace))
|
|
166
|
+
return;
|
|
167
|
+
lazyAddonLoggers.delete(namespace);
|
|
168
|
+
releaseLoggerSafely(loggingModule, modelType);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Runs an addon's `releaseLogger`. A failure must not abort a sweep or leave
|
|
172
|
+
* the registry half-cleared for the next caller, but it is reported rather
|
|
173
|
+
* than swallowed: a leaked reference or a live async handle must not pass for
|
|
174
|
+
* a clean teardown. `subject` names what is being released — a model type, or
|
|
175
|
+
* a namespace for a logger that is only keyed by one — and `during` names the
|
|
176
|
+
* sweep, when the release is part of one.
|
|
177
|
+
*/
|
|
178
|
+
function releaseLoggerSafely(loggingModule, subject, during) {
|
|
179
|
+
try {
|
|
180
|
+
loggingModule.releaseLogger?.();
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
getEngineLogger().warn(`[${subject}] releaseLogger failed${during ? ` during ${during}` : ''}: ${error instanceof Error ? error.message : String(error)}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
70
186
|
export function getPlugin(modelType) {
|
|
71
187
|
return plugins.get(modelType);
|
|
72
188
|
}
|
|
@@ -84,10 +200,12 @@ export function unregisterPlugin(modelType) {
|
|
|
84
200
|
if (!plugin)
|
|
85
201
|
return false;
|
|
86
202
|
const loggingModule = getLoggingModule(plugin);
|
|
203
|
+
const resolverNamespace = getLoggingResolver(plugin) ? plugin.logging?.namespace : undefined;
|
|
87
204
|
plugins.delete(modelType);
|
|
88
205
|
if (loggingModule && !findPluginUsingLoggingModule(loggingModule)) {
|
|
89
206
|
loggingModule.releaseLogger?.();
|
|
90
207
|
}
|
|
208
|
+
releaseLazyAddonLogger(resolverNamespace, modelType);
|
|
91
209
|
return true;
|
|
92
210
|
}
|
|
93
211
|
export function getAllPlugins() {
|
|
@@ -104,16 +222,16 @@ export function clearPlugins() {
|
|
|
104
222
|
loggingModules.set(loggingModule, plugin.modelType);
|
|
105
223
|
}
|
|
106
224
|
}
|
|
225
|
+
const lazyModules = new Map(lazyAddonLoggers);
|
|
107
226
|
plugins.clear();
|
|
227
|
+
lazyAddonLoggers.clear();
|
|
108
228
|
for (const [loggingModule, modelType] of loggingModules) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
getEngineLogger().warn(`[${modelType}] releaseLogger failed during clearPlugins: ${error instanceof Error ? error.message : String(error)}`);
|
|
117
|
-
}
|
|
229
|
+
releaseLoggerSafely(loggingModule, modelType, 'clearPlugins');
|
|
230
|
+
}
|
|
231
|
+
// Loggers wired through a resolver are keyed by namespace rather than held
|
|
232
|
+
// on the plugin, so they need the same sweep. They are named by that
|
|
233
|
+
// namespace, which is not a model type — several plugins can share one.
|
|
234
|
+
for (const [namespace, loggingModule] of lazyModules) {
|
|
235
|
+
releaseLoggerSafely(loggingModule, `namespace ${namespace}`, 'clearPlugins');
|
|
118
236
|
}
|
|
119
237
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
import type { VectorIndexBackend } from '../runtime/vector-index-backend.js';
|
|
3
|
+
/**
|
|
4
|
+
* The native index reports only its effective bit width after a load. Three
|
|
5
|
+
* widths map to one storage mode each; 4 bits is either `q4` or
|
|
6
|
+
* `turbovec-q4`, which the snapshot header distinguishes but the addon does
|
|
7
|
+
* not expose yet, so those loads report no storage mode.
|
|
8
|
+
*/
|
|
9
|
+
export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStorage | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* The vector index feature's only backend: the fabric vector index that the
|
|
12
|
+
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
13
|
+
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
14
|
+
* dimension rule (multiple of 8, at most 1024), which the native index
|
|
15
|
+
* enforces itself.
|
|
16
|
+
*/
|
|
17
|
+
export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { getTurboVecIndexProvider } from './registry.js';
|
|
2
|
+
import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
|
|
3
|
+
// The native index pads short result rows with UINT64_MAX.
|
|
4
|
+
const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
|
|
5
|
+
/**
|
|
6
|
+
* The native index reports only its effective bit width after a load. Three
|
|
7
|
+
* widths map to one storage mode each; 4 bits is either `q4` or
|
|
8
|
+
* `turbovec-q4`, which the snapshot header distinguishes but the addon does
|
|
9
|
+
* not expose yet, so those loads report no storage mode.
|
|
10
|
+
*/
|
|
11
|
+
export function storageFromBitWidth(bitWidth) {
|
|
12
|
+
switch (bitWidth) {
|
|
13
|
+
case 32:
|
|
14
|
+
return VectorIndexStorage.F32;
|
|
15
|
+
case 8:
|
|
16
|
+
return VectorIndexStorage.Q8;
|
|
17
|
+
case 2:
|
|
18
|
+
return VectorIndexStorage.TURBOVEC_Q2;
|
|
19
|
+
default:
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function toHitRows(result) {
|
|
24
|
+
const rows = [];
|
|
25
|
+
for (let row = 0; row < result.m; row++) {
|
|
26
|
+
const hits = [];
|
|
27
|
+
for (let slot = 0; slot < result.k; slot++) {
|
|
28
|
+
const offset = row * result.k + slot;
|
|
29
|
+
const id = result.ids[offset];
|
|
30
|
+
if (id === undefined || id === PADDING_ID)
|
|
31
|
+
break;
|
|
32
|
+
hits.push({ id: id.toString(), score: result.scores[offset] ?? 0 });
|
|
33
|
+
}
|
|
34
|
+
rows.push(hits);
|
|
35
|
+
}
|
|
36
|
+
return rows;
|
|
37
|
+
}
|
|
38
|
+
function openIndex(index, storage) {
|
|
39
|
+
// TurboVec precomputes rotation and codebook state on the first search
|
|
40
|
+
// after a change; running it eagerly keeps that cost out of the first
|
|
41
|
+
// query's latency.
|
|
42
|
+
let needsPrepare = true;
|
|
43
|
+
return {
|
|
44
|
+
get dim() {
|
|
45
|
+
return index.dim;
|
|
46
|
+
},
|
|
47
|
+
get length() {
|
|
48
|
+
return index.length;
|
|
49
|
+
},
|
|
50
|
+
storage,
|
|
51
|
+
add(vectors, ids) {
|
|
52
|
+
index.addWithIds(vectors, ids);
|
|
53
|
+
needsPrepare = true;
|
|
54
|
+
},
|
|
55
|
+
search(queries, k) {
|
|
56
|
+
if (needsPrepare) {
|
|
57
|
+
index.prepare();
|
|
58
|
+
needsPrepare = false;
|
|
59
|
+
}
|
|
60
|
+
return toHitRows(index.search(queries, k));
|
|
61
|
+
},
|
|
62
|
+
contains(id) {
|
|
63
|
+
return index.contains(id);
|
|
64
|
+
},
|
|
65
|
+
remove(id) {
|
|
66
|
+
const removed = index.remove(id);
|
|
67
|
+
if (removed)
|
|
68
|
+
needsPrepare = true;
|
|
69
|
+
return removed;
|
|
70
|
+
},
|
|
71
|
+
write(path) {
|
|
72
|
+
index.write(path);
|
|
73
|
+
},
|
|
74
|
+
dispose() {
|
|
75
|
+
index.dispose();
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The vector index feature's only backend: the fabric vector index that the
|
|
81
|
+
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
82
|
+
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
83
|
+
* dimension rule (multiple of 8, at most 1024), which the native index
|
|
84
|
+
* enforces itself.
|
|
85
|
+
*/
|
|
86
|
+
export function getTurboVecBackend() {
|
|
87
|
+
const provider = getTurboVecIndexProvider();
|
|
88
|
+
if (!provider)
|
|
89
|
+
return undefined;
|
|
90
|
+
return {
|
|
91
|
+
create(dim, storage) {
|
|
92
|
+
return openIndex(provider.create({ dim, storage }), storage);
|
|
93
|
+
},
|
|
94
|
+
load(path) {
|
|
95
|
+
const index = provider.load(path);
|
|
96
|
+
return openIndex(index, storageFromBitWidth(index.bitWidth));
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
}
|
package/dist/registry.js
CHANGED
|
@@ -4,6 +4,7 @@ import { handleUnloadModel } from './handlers/unload-model.js';
|
|
|
4
4
|
import { handleLoggingStream } from './handlers/logging-stream.js';
|
|
5
5
|
import { cancelHandler } from './handlers/cancelHandler.js';
|
|
6
6
|
import { handleRag } from './rag/handler.js';
|
|
7
|
+
import { handleVectorIndex } from './handlers/vector-index.js';
|
|
7
8
|
import { handleDeleteCache } from './handlers/delete-cache.js';
|
|
8
9
|
import { handleDownloadAsset } from './handlers/download-asset.js';
|
|
9
10
|
import { handleGetModelInfo } from './handlers/get-model-info.js';
|
|
@@ -74,6 +75,7 @@ export const registry = {
|
|
|
74
75
|
},
|
|
75
76
|
downloadAsset: { type: 'reply', handler: handleDownloadAsset, supportsProgress: true },
|
|
76
77
|
rag: { type: 'reply', handler: handleRag, supportsProgress: ragSupportsProgress },
|
|
78
|
+
vectorIndex: { type: 'reply', handler: handleVectorIndex },
|
|
77
79
|
finetune: { type: 'reply', handler: handleFinetune, supportsProgress: finetuneSupportsProgress },
|
|
78
80
|
// Stream
|
|
79
81
|
completionStream: {
|
|
@@ -94,6 +96,8 @@ export const registry = {
|
|
|
94
96
|
ocrStream: { type: 'stream', pluginOp: true, handler: pluginStream('ocrStream') },
|
|
95
97
|
diffusionStream: { type: 'stream', pluginOp: true, handler: pluginStream('diffusionStream') },
|
|
96
98
|
audioGenStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioGenStream') },
|
|
99
|
+
audioEditStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioEditStream') },
|
|
100
|
+
audioUnderstand: { type: 'stream', pluginOp: true, handler: pluginStream('audioUnderstand') },
|
|
97
101
|
videoStream: { type: 'stream', pluginOp: true, handler: pluginStream('videoStream') },
|
|
98
102
|
upscaleStream: { type: 'stream', pluginOp: true, handler: pluginStream('upscaleStream') },
|
|
99
103
|
worldStepStream: { type: 'stream', pluginOp: true, handler: pluginStream('worldStepStream') },
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AssessModelFitResult, ModelFitCandidate, ModelFitExecution } from '../../schemas/assess-model-fit.js';
|
|
1
|
+
import type { AssessModelFitResult, ModelFitCandidate, ModelFitExecution, NativeProbeFit } from '../../schemas/assess-model-fit.js';
|
|
2
2
|
import type { SystemResources } from '../../schemas/system-resources.js';
|
|
3
3
|
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
4
4
|
import type { ModelFitPlatform, PlatformCalibration } from './types.js';
|
|
@@ -24,6 +24,15 @@ export interface AssessModelFitOptions {
|
|
|
24
24
|
resolveSharedGpuCalibration?: (platform: ModelFitPlatform, backend: string) => PlatformCalibration | undefined;
|
|
25
25
|
/** Defaults to the generated catalog table; injected in tests. */
|
|
26
26
|
resolveProfile?: ProfileResolver;
|
|
27
|
+
/**
|
|
28
|
+
* The engine's own verdict for the single candidate, from the registry's fit
|
|
29
|
+
* stub. Resolved in the handler, which is where the network is.
|
|
30
|
+
*
|
|
31
|
+
* Only ever set for a one-candidate assessment: the fitter answers about one
|
|
32
|
+
* model against the whole machine, and two such answers cannot be summed into
|
|
33
|
+
* a combined budget the way two byte estimates can.
|
|
34
|
+
*/
|
|
35
|
+
nativeFit?: NativeProbeFit | undefined;
|
|
27
36
|
}
|
|
28
37
|
/**
|
|
29
38
|
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
@@ -168,6 +168,23 @@ export function assessModelFitFromResources(options) {
|
|
|
168
168
|
if (anyFloor) {
|
|
169
169
|
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
170
170
|
}
|
|
171
|
+
// The engine's own fitter, where it reached a verdict, replaces the modelled
|
|
172
|
+
// one: it read this model's tensor list and measured this machine, which is
|
|
173
|
+
// what the coefficients above approximate. The estimator's reasons and
|
|
174
|
+
// assumptions describe the path not taken, so none of them are reported.
|
|
175
|
+
const native = nativeModelResult(options.nativeFit, modelResults);
|
|
176
|
+
if (native) {
|
|
177
|
+
return {
|
|
178
|
+
verdict: native.verdict,
|
|
179
|
+
basis,
|
|
180
|
+
execution,
|
|
181
|
+
evidence: 'native-fit',
|
|
182
|
+
...(budget && { budget }),
|
|
183
|
+
models: [native],
|
|
184
|
+
reasons: ['the engine fitter read the registry description of this model'],
|
|
185
|
+
assumptions: []
|
|
186
|
+
};
|
|
187
|
+
}
|
|
171
188
|
return {
|
|
172
189
|
verdict,
|
|
173
190
|
basis,
|
|
@@ -183,6 +200,32 @@ export function assessModelFitFromResources(options) {
|
|
|
183
200
|
assumptions
|
|
184
201
|
};
|
|
185
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* The per-model result a native fit produces, or `undefined` when the modelled
|
|
205
|
+
* one stands.
|
|
206
|
+
*
|
|
207
|
+
* The probe answers about one model against the whole machine. Two such answers
|
|
208
|
+
* carry no byte demand that could be summed under one budget, so a multi-model
|
|
209
|
+
* assessment keeps the estimator that can aggregate. `unknown` — the probe
|
|
210
|
+
* disabled, the load shape unsupported, the child unusable — is not a verdict
|
|
211
|
+
* and falls through the same way.
|
|
212
|
+
*/
|
|
213
|
+
function nativeModelResult(nativeFit, modelResults) {
|
|
214
|
+
if (!nativeFit || modelResults.length !== 1)
|
|
215
|
+
return undefined;
|
|
216
|
+
if (nativeFit.verdict === 'unknown')
|
|
217
|
+
return undefined;
|
|
218
|
+
const modelled = modelResults[0];
|
|
219
|
+
if (!modelled)
|
|
220
|
+
return undefined;
|
|
221
|
+
return {
|
|
222
|
+
name: modelled.name,
|
|
223
|
+
verdict: nativeFit.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
|
|
224
|
+
evidence: 'native-fit',
|
|
225
|
+
estimatorVersion: nativeFit.estimatorVersion,
|
|
226
|
+
reasons: nativeFit.message === undefined ? [nativeFit.reason] : [nativeFit.reason, nativeFit.message]
|
|
227
|
+
};
|
|
228
|
+
}
|
|
186
229
|
/**
|
|
187
230
|
* Whether every allocation a load makes comes out of the memory the system (or
|
|
188
231
|
* process) budget measures — the precondition for the computed floor to be
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { QVACBlobBinding } from '@qvac/registry-client';
|
|
3
|
+
import type { Logger } from '../../../logging/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* A weightless description of an artifact, served by the registry beside the
|
|
6
|
+
* weights: for a GGUF, the tensor list and the settings with the tokenizer
|
|
7
|
+
* tables stripped and no data section — tens of KB against gigabytes. The
|
|
8
|
+
* fitter reads it exactly as it reads the artifact, so a projection can be made
|
|
9
|
+
* before anything is downloaded.
|
|
10
|
+
*/
|
|
11
|
+
export interface FitStubRef {
|
|
12
|
+
name: string;
|
|
13
|
+
sha256Checksum: string;
|
|
14
|
+
/** Registry coordinates. A constant that carries neither cannot be resolved. */
|
|
15
|
+
registryPath?: string | undefined;
|
|
16
|
+
registrySource?: string | undefined;
|
|
17
|
+
}
|
|
18
|
+
export type FitStubUnavailableReason =
|
|
19
|
+
/** The ref carries no registry coordinates to look up. */
|
|
20
|
+
'unresolvable-ref'
|
|
21
|
+
/** The registry has no entry at those coordinates. */
|
|
22
|
+
| 'not-in-registry'
|
|
23
|
+
/** The entry predates fit blobs, or its description could not be built. */
|
|
24
|
+
| 'no-fit-blob' | 'download-failed'
|
|
25
|
+
/** The lookup and the fetch together did not finish within the budget. */
|
|
26
|
+
| 'timed-out';
|
|
27
|
+
export type FitStubOutcome =
|
|
28
|
+
/** `path` is the caller's to remove once the fitter has read it. */
|
|
29
|
+
{
|
|
30
|
+
status: 'ready';
|
|
31
|
+
path: string;
|
|
32
|
+
bytes: number;
|
|
33
|
+
} | {
|
|
34
|
+
status: 'unavailable';
|
|
35
|
+
reason: FitStubUnavailableReason;
|
|
36
|
+
message?: string;
|
|
37
|
+
};
|
|
38
|
+
export type FitBlobBinding = QVACBlobBinding;
|
|
39
|
+
export interface FitStubEntry {
|
|
40
|
+
fitBlobBinding?: FitBlobBinding | undefined;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
44
|
+
* substitute them rather than mocking modules.
|
|
45
|
+
*/
|
|
46
|
+
export interface FitStubOptions {
|
|
47
|
+
signal?: AbortSignal;
|
|
48
|
+
/** Where stubs are staged. Defaults to `fit-stubs` under the QVAC cache root. */
|
|
49
|
+
cacheDir?: string;
|
|
50
|
+
/** Overall budget for the lookup and the fetch. Defaults to `FIT_STUB_BUDGET_MS`. */
|
|
51
|
+
budgetMs?: number;
|
|
52
|
+
logger?: Logger;
|
|
53
|
+
getEntry?: (registryPath: string, registrySource: string) => Promise<FitStubEntry | null>;
|
|
54
|
+
downloadBlob?: (binding: FitBlobBinding, outputFile: string, signal?: AbortSignal) => Promise<unknown>;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
58
|
+
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
59
|
+
* uncapped wait for the registry view on first use — suit a weights download.
|
|
60
|
+
* Here an unreachable registry is the whole latency of the call, so it is
|
|
61
|
+
* capped. Work still in flight at expiry is abandoned: the download is
|
|
62
|
+
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
63
|
+
* nothing waiting for it.
|
|
64
|
+
*/
|
|
65
|
+
export declare const FIT_STUB_BUDGET_MS = 10000;
|
|
66
|
+
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
67
|
+
export declare function removeStub(stubPath: string): Promise<void>;
|
|
68
|
+
/**
|
|
69
|
+
* Fetches the fit stub for one artifact into its own staging directory and
|
|
70
|
+
* returns its path. The stub is a per-call payload, not a cache: the caller
|
|
71
|
+
* removes it once the fitter has read it, so nothing accumulates under the
|
|
72
|
+
* QVAC root and no cleanup path has to know about it.
|
|
73
|
+
*
|
|
74
|
+
* Never throws: a missing entry, an entry without a description, a failed or
|
|
75
|
+
* corrupt download and an expired budget are all `unavailable`, because a
|
|
76
|
+
* pre-download assessment has to survive an offline caller and an older
|
|
77
|
+
* registry record.
|
|
78
|
+
*
|
|
79
|
+
* Only the artifact named by `ref` is fetched. A split model's shards each
|
|
80
|
+
* carry their own binding, and the fitter needs the whole set laid out under
|
|
81
|
+
* llama's shard naming to read any of it — assembling that is not done here.
|
|
82
|
+
*/
|
|
83
|
+
export declare function fetchFitStub(ref: FitStubRef, options?: FitStubOptions): Promise<FitStubOutcome>;
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { AbortController } from 'bare-abort-controller';
|
|
4
|
+
import { getCacheDir } from '../../../utils/cache/paths.js';
|
|
5
|
+
import { calculateFileChecksum } from '../../../utils/checksum.js';
|
|
6
|
+
import { getEngineLogger } from '../../../logging/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
9
|
+
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
10
|
+
* uncapped wait for the registry view on first use — suit a weights download.
|
|
11
|
+
* Here an unreachable registry is the whole latency of the call, so it is
|
|
12
|
+
* capped. Work still in flight at expiry is abandoned: the download is
|
|
13
|
+
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
14
|
+
* nothing waiting for it.
|
|
15
|
+
*/
|
|
16
|
+
export const FIT_STUB_BUDGET_MS = 10_000;
|
|
17
|
+
async function defaultGetEntry(registryPath, registrySource) {
|
|
18
|
+
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
19
|
+
const client = await getRegistryClient();
|
|
20
|
+
return client.getModel(registryPath, registrySource);
|
|
21
|
+
}
|
|
22
|
+
async function defaultDownloadBlob(binding, outputFile, signal) {
|
|
23
|
+
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
24
|
+
const client = await getRegistryClient();
|
|
25
|
+
return client.downloadBlob(binding, {
|
|
26
|
+
outputFile,
|
|
27
|
+
maxRetries: 1,
|
|
28
|
+
...(signal !== undefined && { signal })
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
32
|
+
export async function removeStub(stubPath) {
|
|
33
|
+
await fsPromises.rm(path.dirname(stubPath), { recursive: true, force: true }).catch(() => { });
|
|
34
|
+
}
|
|
35
|
+
function unavailable(reason, message) {
|
|
36
|
+
return message === undefined
|
|
37
|
+
? { status: 'unavailable', reason }
|
|
38
|
+
: { status: 'unavailable', reason, message };
|
|
39
|
+
}
|
|
40
|
+
function describe(error) {
|
|
41
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Fetches the fit stub for one artifact into its own staging directory and
|
|
45
|
+
* returns its path. The stub is a per-call payload, not a cache: the caller
|
|
46
|
+
* removes it once the fitter has read it, so nothing accumulates under the
|
|
47
|
+
* QVAC root and no cleanup path has to know about it.
|
|
48
|
+
*
|
|
49
|
+
* Never throws: a missing entry, an entry without a description, a failed or
|
|
50
|
+
* corrupt download and an expired budget are all `unavailable`, because a
|
|
51
|
+
* pre-download assessment has to survive an offline caller and an older
|
|
52
|
+
* registry record.
|
|
53
|
+
*
|
|
54
|
+
* Only the artifact named by `ref` is fetched. A split model's shards each
|
|
55
|
+
* carry their own binding, and the fitter needs the whole set laid out under
|
|
56
|
+
* llama's shard naming to read any of it — assembling that is not done here.
|
|
57
|
+
*/
|
|
58
|
+
export async function fetchFitStub(ref, options = {}) {
|
|
59
|
+
const budgetMs = options.budgetMs ?? FIT_STUB_BUDGET_MS;
|
|
60
|
+
const controller = new AbortController();
|
|
61
|
+
const onAbort = () => controller.abort(new Error('fit stub fetch aborted by the caller'));
|
|
62
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
63
|
+
let timer;
|
|
64
|
+
const expiry = new Promise((resolve) => {
|
|
65
|
+
timer = setTimeout(() => {
|
|
66
|
+
controller.abort(new Error('fit stub budget expired'));
|
|
67
|
+
resolve(unavailable('timed-out', `no fit stub for ${ref.name} from the registry within ${budgetMs}ms`));
|
|
68
|
+
}, budgetMs);
|
|
69
|
+
});
|
|
70
|
+
try {
|
|
71
|
+
return await Promise.race([fetchWithin(ref, options, controller.signal), expiry]);
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
clearTimeout(timer);
|
|
75
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async function fetchWithin(ref, options, signal) {
|
|
79
|
+
const logger = options.logger ?? getEngineLogger();
|
|
80
|
+
try {
|
|
81
|
+
const { registryPath, registrySource } = ref;
|
|
82
|
+
if (!registryPath || !registrySource)
|
|
83
|
+
return unavailable('unresolvable-ref');
|
|
84
|
+
const getEntry = options.getEntry ?? defaultGetEntry;
|
|
85
|
+
const entry = await getEntry(registryPath, registrySource);
|
|
86
|
+
if (entry === null || entry === undefined)
|
|
87
|
+
return unavailable('not-in-registry');
|
|
88
|
+
const binding = entry.fitBlobBinding;
|
|
89
|
+
if (binding === undefined || binding === null)
|
|
90
|
+
return unavailable('no-fit-blob');
|
|
91
|
+
if (signal.aborted)
|
|
92
|
+
return unavailable('timed-out');
|
|
93
|
+
// One directory per fetch, so two assessments of the same model never share
|
|
94
|
+
// a file that one of them is about to remove.
|
|
95
|
+
const root = options.cacheDir ?? getCacheDir('fit-stubs');
|
|
96
|
+
await fsPromises.mkdir(root, { recursive: true });
|
|
97
|
+
const dir = await fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
98
|
+
const dest = path.join(dir, `${binding.sha256}.gguf`);
|
|
99
|
+
const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
|
|
100
|
+
let verified = false;
|
|
101
|
+
try {
|
|
102
|
+
await downloadBlob(binding, dest, signal);
|
|
103
|
+
const bytes = fs.statSync(dest).size;
|
|
104
|
+
if (bytes !== binding.byteLength) {
|
|
105
|
+
return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
|
|
106
|
+
}
|
|
107
|
+
// The record binds the description by digest; a stub that reads back
|
|
108
|
+
// differently is not the description the fitter should answer for.
|
|
109
|
+
const digest = await calculateFileChecksum(dest);
|
|
110
|
+
if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
|
|
111
|
+
return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
|
|
112
|
+
}
|
|
113
|
+
// Past the budget nobody is waiting for this result, so the stub must
|
|
114
|
+
// not be marked as kept.
|
|
115
|
+
if (signal.aborted)
|
|
116
|
+
return unavailable('timed-out');
|
|
117
|
+
verified = true;
|
|
118
|
+
return { status: 'ready', path: dest, bytes };
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
return unavailable('download-failed', describe(error));
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
// A short, corrupt, half-written or late stub must not outlive the call.
|
|
125
|
+
if (!verified)
|
|
126
|
+
await removeStub(dest);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
|
|
131
|
+
return unavailable('download-failed', describe(error));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { fetchFitStub, type FitBlobBinding, type FitStubEntry, type FitStubOptions, type FitStubOutcome, type FitStubRef, type FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
2
|
+
export { projectFitFromStub, type StubFitInput, type StubFitOptions, type StubFitOutcome } from './project-fit-from-stub.js';
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { CanonicalModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { type FitStubOptions, type FitStubRef, type FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
3
|
+
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
|
|
4
|
+
export interface StubFitInput {
|
|
5
|
+
model: FitStubRef;
|
|
6
|
+
modelType: CanonicalModelType;
|
|
7
|
+
/** The load the caller intends, in the spelling the engine's config uses. */
|
|
8
|
+
modelConfig: unknown;
|
|
9
|
+
}
|
|
10
|
+
export type StubFitOutcome = {
|
|
11
|
+
status: 'projected';
|
|
12
|
+
fit: AdvisoryFitOutcome;
|
|
13
|
+
} | {
|
|
14
|
+
status: 'no-stub';
|
|
15
|
+
reason: FitStubUnavailableReason;
|
|
16
|
+
message?: string;
|
|
17
|
+
};
|
|
18
|
+
export interface StubFitOptions {
|
|
19
|
+
stub?: FitStubOptions;
|
|
20
|
+
fit?: AdvisoryFitOptions;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Projects a load from the registry's fit stub, before the weights exist
|
|
24
|
+
* locally.
|
|
25
|
+
*
|
|
26
|
+
* The stub is an ordinary GGUF as far as the fitter is concerned, so this is
|
|
27
|
+
* the same probe `loadModel` runs, pointed at a file that is tens of KB instead
|
|
28
|
+
* of gigabytes. It inherits that probe's fail-open contract: every failure is
|
|
29
|
+
* an `unknown` verdict or a `no-stub` outcome, never a throw.
|
|
30
|
+
*/
|
|
31
|
+
export declare function projectFitFromStub(input: StubFitInput, options?: StubFitOptions): Promise<StubFitOutcome>;
|