@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
package/dist/plugins/registry.js
CHANGED
|
@@ -5,9 +5,41 @@ import { createAddonLoggerCallback } from '../logging/addon.js';
|
|
|
5
5
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
6
|
import { formatZodError } from '../utils/zod-error.js';
|
|
7
7
|
const plugins = new Map();
|
|
8
|
+
/** Addon logging modules wired through a resolver, keyed by namespace. */
|
|
9
|
+
const lazyAddonLoggers = new Map();
|
|
10
|
+
/** In-flight resolutions, so concurrent loads wire a namespace exactly once. */
|
|
11
|
+
const pendingAddonLoggers = new Map();
|
|
12
|
+
function getLoggingResolver(plugin) {
|
|
13
|
+
const declared = plugin.logging?.module;
|
|
14
|
+
return typeof declared === 'function' ? declared : undefined;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The module a plugin supplied directly, or `undefined` when it supplied a
|
|
18
|
+
* resolver. Resolver-based plugins are therefore invisible to
|
|
19
|
+
* registration-time shape validation, to the shared-module dedupe, and to the
|
|
20
|
+
* eager release sweeps; `ensureAddonLoggerReady` and `lazyAddonLoggers` cover
|
|
21
|
+
* them instead, keyed by namespace rather than held on the plugin.
|
|
22
|
+
*/
|
|
8
23
|
function getLoggingModule(plugin) {
|
|
24
|
+
if (getLoggingResolver(plugin))
|
|
25
|
+
return undefined;
|
|
9
26
|
return plugin.logging?.module;
|
|
10
27
|
}
|
|
28
|
+
/** CommonJS addons resolve through `import()` as a namespace with `default`. */
|
|
29
|
+
function unwrapLoggingModule(resolved) {
|
|
30
|
+
if (resolved && typeof resolved === 'object' && 'default' in resolved) {
|
|
31
|
+
const inner = resolved.default;
|
|
32
|
+
if (inner && typeof inner.setLogger === 'function')
|
|
33
|
+
return inner;
|
|
34
|
+
}
|
|
35
|
+
return resolved;
|
|
36
|
+
}
|
|
37
|
+
function assertLoggingModuleShape(modelType, candidate) {
|
|
38
|
+
if (!candidate || typeof candidate['setLogger'] !== 'function') {
|
|
39
|
+
throw new PluginLoggingInvalidError(modelType, 'logging.module must have a setLogger(callback) function');
|
|
40
|
+
}
|
|
41
|
+
return candidate;
|
|
42
|
+
}
|
|
11
43
|
function findPluginUsingLoggingModule(loggingModule) {
|
|
12
44
|
return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
|
|
13
45
|
}
|
|
@@ -42,12 +74,11 @@ export function registerPlugin(plugin) {
|
|
|
42
74
|
if (pluginProvidingTurboVecIndex) {
|
|
43
75
|
throw new PluginDefinitionInvalidError(plugin.modelType, `plugin "${plugin.modelType}" cannot provide turbovecIndexProvider because plugin "${pluginProvidingTurboVecIndex.modelType}" already provides it`);
|
|
44
76
|
}
|
|
45
|
-
// Validate logging module shape if provided
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
77
|
+
// Validate logging module shape if provided. A resolver is checked when it
|
|
78
|
+
// runs instead: calling it here to inspect its result would load the addon,
|
|
79
|
+
// which is the whole thing registration is meant to avoid.
|
|
80
|
+
if (plugin.logging?.module && !getLoggingResolver(plugin)) {
|
|
81
|
+
assertLoggingModuleShape(plugin.modelType, plugin.logging.module);
|
|
51
82
|
}
|
|
52
83
|
const loggingModule = getLoggingModule(plugin);
|
|
53
84
|
const pluginUsingLoggingModule = loggingModule
|
|
@@ -67,6 +98,91 @@ export function registerPlugins(pluginList) {
|
|
|
67
98
|
registerPlugin(plugin);
|
|
68
99
|
}
|
|
69
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Wires the addon logger of a plugin that supplied a resolver, loading the
|
|
103
|
+
* addon on the way. Call it once a model of this type is actually being
|
|
104
|
+
* created: it is the point where the addon is needed anyway, so a host
|
|
105
|
+
* missing that addon's platform package fails the load it asked for instead
|
|
106
|
+
* of failing every plugin registration at startup.
|
|
107
|
+
*
|
|
108
|
+
* A no-op for plugins that passed their module directly — those are already
|
|
109
|
+
* wired by `registerPlugin` — and for a namespace that is already wired,
|
|
110
|
+
* which is how two plugins over one addon (whisper and Parakeet over ASR)
|
|
111
|
+
* share a single `setLogger` call. Also a no-op when the registry was cleared
|
|
112
|
+
* while the addon was loading, so a shutdown that races a load leaves no
|
|
113
|
+
* logger behind.
|
|
114
|
+
*/
|
|
115
|
+
export async function ensureAddonLoggerReady(plugin) {
|
|
116
|
+
const resolver = getLoggingResolver(plugin);
|
|
117
|
+
const namespace = plugin.logging?.namespace;
|
|
118
|
+
if (!resolver || !namespace)
|
|
119
|
+
return;
|
|
120
|
+
if (lazyAddonLoggers.has(namespace))
|
|
121
|
+
return;
|
|
122
|
+
const pending = pendingAddonLoggers.get(namespace);
|
|
123
|
+
if (pending)
|
|
124
|
+
return pending;
|
|
125
|
+
const wiring = (async () => {
|
|
126
|
+
const resolved = unwrapLoggingModule(await resolver());
|
|
127
|
+
// Loading the addon takes real time, and the registry can be torn down
|
|
128
|
+
// while this is parked: `runCleanup` calls `clearRegistries()`
|
|
129
|
+
// synchronously and only then awaits `unloadAllModels()`, so a sweep can
|
|
130
|
+
// pass between the call above and this line. Wiring now would hand the
|
|
131
|
+
// addon a callback nothing will release — on Expo, a `js_ref_t` leaked
|
|
132
|
+
// into a dying isolate, which the next worklet's first `setLogger` trips
|
|
133
|
+
// over. Nothing has been wired yet, so dropping the resolved module is
|
|
134
|
+
// the whole cleanup.
|
|
135
|
+
if (!isNamespaceClaimed(namespace))
|
|
136
|
+
return;
|
|
137
|
+
const loggingModule = assertLoggingModuleShape(plugin.modelType, resolved);
|
|
138
|
+
for (const [wiredNamespace, wiredModule] of lazyAddonLoggers) {
|
|
139
|
+
if (wiredModule === loggingModule && wiredNamespace !== namespace) {
|
|
140
|
+
throw new PluginLoggingInvalidError(plugin.modelType, 'plugins sharing logging.module must use the same namespace');
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
loggingModule.setLogger(createAddonLoggerCallback(namespace));
|
|
144
|
+
lazyAddonLoggers.set(namespace, loggingModule);
|
|
145
|
+
})();
|
|
146
|
+
pendingAddonLoggers.set(namespace, wiring);
|
|
147
|
+
try {
|
|
148
|
+
await wiring;
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
pendingAddonLoggers.delete(namespace);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** Whether a registered resolver-based plugin still claims this namespace. */
|
|
155
|
+
function isNamespaceClaimed(namespace) {
|
|
156
|
+
return Array.from(plugins.values()).some((candidate) => getLoggingResolver(candidate) && candidate.logging?.namespace === namespace);
|
|
157
|
+
}
|
|
158
|
+
/** Releases a lazily-wired addon logger once no registered plugin claims it. */
|
|
159
|
+
function releaseLazyAddonLogger(namespace, modelType) {
|
|
160
|
+
if (namespace === undefined)
|
|
161
|
+
return;
|
|
162
|
+
const loggingModule = lazyAddonLoggers.get(namespace);
|
|
163
|
+
if (!loggingModule)
|
|
164
|
+
return;
|
|
165
|
+
if (isNamespaceClaimed(namespace))
|
|
166
|
+
return;
|
|
167
|
+
lazyAddonLoggers.delete(namespace);
|
|
168
|
+
releaseLoggerSafely(loggingModule, modelType);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Runs an addon's `releaseLogger`. A failure must not abort a sweep or leave
|
|
172
|
+
* the registry half-cleared for the next caller, but it is reported rather
|
|
173
|
+
* than swallowed: a leaked reference or a live async handle must not pass for
|
|
174
|
+
* a clean teardown. `subject` names what is being released — a model type, or
|
|
175
|
+
* a namespace for a logger that is only keyed by one — and `during` names the
|
|
176
|
+
* sweep, when the release is part of one.
|
|
177
|
+
*/
|
|
178
|
+
function releaseLoggerSafely(loggingModule, subject, during) {
|
|
179
|
+
try {
|
|
180
|
+
loggingModule.releaseLogger?.();
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
getEngineLogger().warn(`[${subject}] releaseLogger failed${during ? ` during ${during}` : ''}: ${error instanceof Error ? error.message : String(error)}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
70
186
|
export function getPlugin(modelType) {
|
|
71
187
|
return plugins.get(modelType);
|
|
72
188
|
}
|
|
@@ -84,10 +200,12 @@ export function unregisterPlugin(modelType) {
|
|
|
84
200
|
if (!plugin)
|
|
85
201
|
return false;
|
|
86
202
|
const loggingModule = getLoggingModule(plugin);
|
|
203
|
+
const resolverNamespace = getLoggingResolver(plugin) ? plugin.logging?.namespace : undefined;
|
|
87
204
|
plugins.delete(modelType);
|
|
88
205
|
if (loggingModule && !findPluginUsingLoggingModule(loggingModule)) {
|
|
89
206
|
loggingModule.releaseLogger?.();
|
|
90
207
|
}
|
|
208
|
+
releaseLazyAddonLogger(resolverNamespace, modelType);
|
|
91
209
|
return true;
|
|
92
210
|
}
|
|
93
211
|
export function getAllPlugins() {
|
|
@@ -104,16 +222,16 @@ export function clearPlugins() {
|
|
|
104
222
|
loggingModules.set(loggingModule, plugin.modelType);
|
|
105
223
|
}
|
|
106
224
|
}
|
|
225
|
+
const lazyModules = new Map(lazyAddonLoggers);
|
|
107
226
|
plugins.clear();
|
|
227
|
+
lazyAddonLoggers.clear();
|
|
108
228
|
for (const [loggingModule, modelType] of loggingModules) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
getEngineLogger().warn(`[${modelType}] releaseLogger failed during clearPlugins: ${error instanceof Error ? error.message : String(error)}`);
|
|
117
|
-
}
|
|
229
|
+
releaseLoggerSafely(loggingModule, modelType, 'clearPlugins');
|
|
230
|
+
}
|
|
231
|
+
// Loggers wired through a resolver are keyed by namespace rather than held
|
|
232
|
+
// on the plugin, so they need the same sweep. They are named by that
|
|
233
|
+
// namespace, which is not a model type — several plugins can share one.
|
|
234
|
+
for (const [namespace, loggingModule] of lazyModules) {
|
|
235
|
+
releaseLoggerSafely(loggingModule, `namespace ${namespace}`, 'clearPlugins');
|
|
118
236
|
}
|
|
119
237
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
import type { VectorIndexBackend } from '../runtime/vector-index-backend.js';
|
|
3
|
+
/**
|
|
4
|
+
* The native index reports only its effective bit width after a load. Three
|
|
5
|
+
* widths map to one storage mode each; 4 bits is either `q4` or
|
|
6
|
+
* `turbovec-q4`, which the snapshot header distinguishes but the addon does
|
|
7
|
+
* not expose yet, so those loads report no storage mode.
|
|
8
|
+
*/
|
|
9
|
+
export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStorage | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* The vector index feature's only backend: the fabric vector index that the
|
|
12
|
+
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
13
|
+
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
14
|
+
* dimension rule (multiple of 8, at most 1024), which the native index
|
|
15
|
+
* enforces itself.
|
|
16
|
+
*/
|
|
17
|
+
export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { getTurboVecIndexProvider } from './registry.js';
|
|
2
|
+
import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
|
|
3
|
+
// The native index pads short result rows with UINT64_MAX.
|
|
4
|
+
const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
|
|
5
|
+
/**
|
|
6
|
+
* The native index reports only its effective bit width after a load. Three
|
|
7
|
+
* widths map to one storage mode each; 4 bits is either `q4` or
|
|
8
|
+
* `turbovec-q4`, which the snapshot header distinguishes but the addon does
|
|
9
|
+
* not expose yet, so those loads report no storage mode.
|
|
10
|
+
*/
|
|
11
|
+
export function storageFromBitWidth(bitWidth) {
|
|
12
|
+
switch (bitWidth) {
|
|
13
|
+
case 32:
|
|
14
|
+
return VectorIndexStorage.F32;
|
|
15
|
+
case 8:
|
|
16
|
+
return VectorIndexStorage.Q8;
|
|
17
|
+
case 2:
|
|
18
|
+
return VectorIndexStorage.TURBOVEC_Q2;
|
|
19
|
+
default:
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function toHitRows(result) {
|
|
24
|
+
const rows = [];
|
|
25
|
+
for (let row = 0; row < result.m; row++) {
|
|
26
|
+
const hits = [];
|
|
27
|
+
for (let slot = 0; slot < result.k; slot++) {
|
|
28
|
+
const offset = row * result.k + slot;
|
|
29
|
+
const id = result.ids[offset];
|
|
30
|
+
if (id === undefined || id === PADDING_ID)
|
|
31
|
+
break;
|
|
32
|
+
hits.push({ id: id.toString(), score: result.scores[offset] ?? 0 });
|
|
33
|
+
}
|
|
34
|
+
rows.push(hits);
|
|
35
|
+
}
|
|
36
|
+
return rows;
|
|
37
|
+
}
|
|
38
|
+
function openIndex(index, storage) {
|
|
39
|
+
// TurboVec precomputes rotation and codebook state on the first search
|
|
40
|
+
// after a change; running it eagerly keeps that cost out of the first
|
|
41
|
+
// query's latency.
|
|
42
|
+
let needsPrepare = true;
|
|
43
|
+
return {
|
|
44
|
+
get dim() {
|
|
45
|
+
return index.dim;
|
|
46
|
+
},
|
|
47
|
+
get length() {
|
|
48
|
+
return index.length;
|
|
49
|
+
},
|
|
50
|
+
storage,
|
|
51
|
+
add(vectors, ids) {
|
|
52
|
+
index.addWithIds(vectors, ids);
|
|
53
|
+
needsPrepare = true;
|
|
54
|
+
},
|
|
55
|
+
search(queries, k) {
|
|
56
|
+
if (needsPrepare) {
|
|
57
|
+
index.prepare();
|
|
58
|
+
needsPrepare = false;
|
|
59
|
+
}
|
|
60
|
+
return toHitRows(index.search(queries, k));
|
|
61
|
+
},
|
|
62
|
+
contains(id) {
|
|
63
|
+
return index.contains(id);
|
|
64
|
+
},
|
|
65
|
+
remove(id) {
|
|
66
|
+
const removed = index.remove(id);
|
|
67
|
+
if (removed)
|
|
68
|
+
needsPrepare = true;
|
|
69
|
+
return removed;
|
|
70
|
+
},
|
|
71
|
+
write(path) {
|
|
72
|
+
index.write(path);
|
|
73
|
+
},
|
|
74
|
+
dispose() {
|
|
75
|
+
index.dispose();
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The vector index feature's only backend: the fabric vector index that the
|
|
81
|
+
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
82
|
+
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
83
|
+
* dimension rule (multiple of 8, at most 1024), which the native index
|
|
84
|
+
* enforces itself.
|
|
85
|
+
*/
|
|
86
|
+
export function getTurboVecBackend() {
|
|
87
|
+
const provider = getTurboVecIndexProvider();
|
|
88
|
+
if (!provider)
|
|
89
|
+
return undefined;
|
|
90
|
+
return {
|
|
91
|
+
create(dim, storage) {
|
|
92
|
+
return openIndex(provider.create({ dim, storage }), storage);
|
|
93
|
+
},
|
|
94
|
+
load(path) {
|
|
95
|
+
const index = provider.load(path);
|
|
96
|
+
return openIndex(index, storageFromBitWidth(index.bitWidth));
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
}
|
package/dist/registry.js
CHANGED
|
@@ -4,6 +4,7 @@ import { handleUnloadModel } from './handlers/unload-model.js';
|
|
|
4
4
|
import { handleLoggingStream } from './handlers/logging-stream.js';
|
|
5
5
|
import { cancelHandler } from './handlers/cancelHandler.js';
|
|
6
6
|
import { handleRag } from './rag/handler.js';
|
|
7
|
+
import { handleVectorIndex } from './handlers/vector-index.js';
|
|
7
8
|
import { handleDeleteCache } from './handlers/delete-cache.js';
|
|
8
9
|
import { handleDownloadAsset } from './handlers/download-asset.js';
|
|
9
10
|
import { handleGetModelInfo } from './handlers/get-model-info.js';
|
|
@@ -74,6 +75,7 @@ export const registry = {
|
|
|
74
75
|
},
|
|
75
76
|
downloadAsset: { type: 'reply', handler: handleDownloadAsset, supportsProgress: true },
|
|
76
77
|
rag: { type: 'reply', handler: handleRag, supportsProgress: ragSupportsProgress },
|
|
78
|
+
vectorIndex: { type: 'reply', handler: handleVectorIndex },
|
|
77
79
|
finetune: { type: 'reply', handler: handleFinetune, supportsProgress: finetuneSupportsProgress },
|
|
78
80
|
// Stream
|
|
79
81
|
completionStream: {
|
|
@@ -94,6 +96,8 @@ export const registry = {
|
|
|
94
96
|
ocrStream: { type: 'stream', pluginOp: true, handler: pluginStream('ocrStream') },
|
|
95
97
|
diffusionStream: { type: 'stream', pluginOp: true, handler: pluginStream('diffusionStream') },
|
|
96
98
|
audioGenStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioGenStream') },
|
|
99
|
+
audioEditStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioEditStream') },
|
|
100
|
+
audioUnderstand: { type: 'stream', pluginOp: true, handler: pluginStream('audioUnderstand') },
|
|
97
101
|
videoStream: { type: 'stream', pluginOp: true, handler: pluginStream('videoStream') },
|
|
98
102
|
upscaleStream: { type: 'stream', pluginOp: true, handler: pluginStream('upscaleStream') },
|
|
99
103
|
worldStepStream: { type: 'stream', pluginOp: true, handler: pluginStream('worldStepStream') },
|
|
@@ -2,6 +2,7 @@ import { getModelResourceProfile } from '../../models/registry/resource-profiles
|
|
|
2
2
|
import { getGpuCalibration, getSharedGpuCalibration } from './calibration/index.js';
|
|
3
3
|
import { estimateLlm } from './estimators/llm.js';
|
|
4
4
|
import { estimateWhisper } from './estimators/whisper.js';
|
|
5
|
+
import { computeFloor, FLOOR_VERSION } from './floor.js';
|
|
5
6
|
const GIB = 1024 * 1024 * 1024;
|
|
6
7
|
/** Engines phase 1 can estimate, and which estimator owns each. */
|
|
7
8
|
const ESTIMATORS = {
|
|
@@ -10,6 +11,12 @@ const ESTIMATORS = {
|
|
|
10
11
|
'whispercpp-transcription': estimateWhisper
|
|
11
12
|
};
|
|
12
13
|
const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
|
|
14
|
+
// Every allocation, GPU included, is system RAM: the system budget always bounds the load.
|
|
15
|
+
const UNIFIED_MEMORY_PLATFORMS = [
|
|
16
|
+
'darwin-arm64',
|
|
17
|
+
'android-arm64',
|
|
18
|
+
'ios-arm64'
|
|
19
|
+
];
|
|
13
20
|
/**
|
|
14
21
|
* Platforms whose fixture describes CPU-resident execution (`device: 'cpu'`),
|
|
15
22
|
* so a reported GPU sends the host through `resolveGpuPlacement` instead.
|
|
@@ -98,7 +105,12 @@ export function assessModelFitFromResources(options) {
|
|
|
98
105
|
else if (effectiveCalibration.measuredAt) {
|
|
99
106
|
assumptions.push(calibrationAssumption(platform, effectiveCalibration));
|
|
100
107
|
}
|
|
101
|
-
|
|
108
|
+
// Without coefficients, the computed floor still says something — but only
|
|
109
|
+
// where the model executes out of the memory the budget measures. A discrete
|
|
110
|
+
// card holds the weights in its own memory, so the system budget bounds
|
|
111
|
+
// nothing there and the floor has no budget to be compared against.
|
|
112
|
+
const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
|
|
113
|
+
const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode, floorApplies));
|
|
102
114
|
for (const { result } of evaluated) {
|
|
103
115
|
for (const assumption of result.assumptions ?? []) {
|
|
104
116
|
if (!assumptions.includes(assumption))
|
|
@@ -106,19 +118,37 @@ export function assessModelFitFromResources(options) {
|
|
|
106
118
|
}
|
|
107
119
|
}
|
|
108
120
|
const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
.filter((result) => {
|
|
121
|
+
const results = evaluated.map(({ result }) => result);
|
|
122
|
+
const estimates = results.filter((result) => {
|
|
112
123
|
return result.kind === 'estimate';
|
|
113
124
|
});
|
|
114
|
-
const anyUnknown =
|
|
115
|
-
const
|
|
125
|
+
const anyUnknown = results.some((result) => result.kind === 'unknown');
|
|
126
|
+
const anyFloor = results.some((result) => result.kind === 'floor');
|
|
127
|
+
// A two-sided bound needs every candidate estimated; a floor needs every
|
|
128
|
+
// candidate to have at least a floor. One model with neither leaves the set
|
|
129
|
+
// with no evidence at all.
|
|
130
|
+
const combined = anyUnknown || anyFloor ? undefined : aggregate(estimates, execution);
|
|
131
|
+
const combinedFloor = anyUnknown ? undefined : aggregateFloor(results, execution);
|
|
132
|
+
// Absent when a candidate rests on nothing: the set then has no single kind
|
|
133
|
+
// of evidence to name, and an `unknown` with `evidence` present stays a
|
|
134
|
+
// near-miss or an uncalibrated floor rather than a missing model.
|
|
135
|
+
const evidence = anyUnknown
|
|
136
|
+
? undefined
|
|
137
|
+
: anyFloor
|
|
138
|
+
? 'computed-only'
|
|
139
|
+
: estimates.length > 0
|
|
140
|
+
? 'calibration'
|
|
141
|
+
: undefined;
|
|
116
142
|
if (anyUnknown) {
|
|
117
143
|
reasons.push('at least one model could not be estimated, so the combined verdict is unknown');
|
|
118
144
|
}
|
|
119
|
-
const verdict = !budget
|
|
145
|
+
const verdict = !budget
|
|
120
146
|
? 'unknown'
|
|
121
|
-
:
|
|
147
|
+
: combined
|
|
148
|
+
? verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy)
|
|
149
|
+
: combinedFloor !== undefined
|
|
150
|
+
? floorVerdict(combinedFloor, deviceBudgets ?? [budget])
|
|
151
|
+
: 'unknown';
|
|
122
152
|
if (onDevice && combined && alsoBoundBy) {
|
|
123
153
|
reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
|
|
124
154
|
}
|
|
@@ -130,92 +160,110 @@ export function assessModelFitFromResources(options) {
|
|
|
130
160
|
? 'all models counted as resident with every working peak added'
|
|
131
161
|
: 'all models counted as resident with only the largest working peak added');
|
|
132
162
|
}
|
|
163
|
+
else if (budget && combinedFloor !== undefined) {
|
|
164
|
+
reasons.push(execution === 'concurrent'
|
|
165
|
+
? 'all models counted as resident with their floors summed and every calibrated working peak added'
|
|
166
|
+
: 'all models counted as resident with their floors summed and only the largest calibrated working peak added');
|
|
167
|
+
}
|
|
168
|
+
if (anyFloor) {
|
|
169
|
+
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
170
|
+
}
|
|
133
171
|
return {
|
|
134
172
|
verdict,
|
|
135
173
|
basis,
|
|
136
174
|
execution,
|
|
175
|
+
...(evidence && { evidence }),
|
|
137
176
|
...(budget && { budget }),
|
|
138
177
|
...(combined && {
|
|
139
178
|
estimate: { lowerBoundBytes: combined.lower, upperBoundBytes: combined.upper }
|
|
140
179
|
}),
|
|
180
|
+
...(anyFloor && combinedFloor !== undefined && { floorBytes: combinedFloor }),
|
|
141
181
|
models: modelResults,
|
|
142
182
|
reasons,
|
|
143
183
|
assumptions
|
|
144
184
|
};
|
|
145
185
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
if (
|
|
164
|
-
return
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
186
|
+
/**
|
|
187
|
+
* Whether every allocation a load makes comes out of the memory the system (or
|
|
188
|
+
* process) budget measures — the precondition for the computed floor to be
|
|
189
|
+
* compared against that budget.
|
|
190
|
+
*
|
|
191
|
+
* True on unified-memory platforms whatever the collector reports, and
|
|
192
|
+
* elsewhere when no usable GPU is present or every usable GPU allocates out of
|
|
193
|
+
* system RAM. A card with its own memory, or a device whose class cannot be
|
|
194
|
+
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
195
|
+
* cannot see them.
|
|
196
|
+
*/
|
|
197
|
+
function boundBySystemMemory(resources, platform) {
|
|
198
|
+
if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
|
|
199
|
+
return true;
|
|
200
|
+
const gpus = usableGpus(resources);
|
|
201
|
+
if (gpus.length === 0)
|
|
202
|
+
return true;
|
|
203
|
+
if (platform && gpus.some((gpu) => integratedIsIndistinguishable(gpu, platform)))
|
|
204
|
+
return false;
|
|
205
|
+
return gpus.every(allocatesFromSystemMemory);
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* One candidate's evaluation: the calibrated estimate when the coefficients
|
|
209
|
+
* describe this load, otherwise the computed floor where a floor can be
|
|
210
|
+
* compared at all, otherwise `unknown`. Whatever kept the estimate from
|
|
211
|
+
* forming stays in the reasons, so a floor verdict still says why it is only
|
|
212
|
+
* a floor.
|
|
213
|
+
*/
|
|
214
|
+
function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode, floorApplies) {
|
|
175
215
|
const profile = resolveProfile(candidate.model.sha256Checksum);
|
|
176
216
|
if (!profile) {
|
|
177
|
-
return {
|
|
178
|
-
candidate,
|
|
179
|
-
result: {
|
|
180
|
-
kind: 'unknown',
|
|
181
|
-
estimatorVersion: 'none',
|
|
182
|
-
reasons: ['no resource profile in the catalog for this checksum']
|
|
183
|
-
}
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
const estimator = ESTIMATORS[profile.engine];
|
|
187
|
-
if (!estimator) {
|
|
188
|
-
return {
|
|
189
|
-
candidate,
|
|
190
|
-
result: {
|
|
191
|
-
kind: 'unknown',
|
|
192
|
-
estimatorVersion: 'none',
|
|
193
|
-
reasons: [`engine '${profile.engine}' has no estimator in this phase`]
|
|
194
|
-
}
|
|
195
|
-
};
|
|
217
|
+
return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
|
|
196
218
|
}
|
|
197
219
|
const extra = extraArtifactBytes(candidate, resolveProfile);
|
|
198
220
|
if (extra === undefined) {
|
|
199
221
|
return {
|
|
200
222
|
candidate,
|
|
201
|
-
result:
|
|
202
|
-
kind: 'unknown',
|
|
203
|
-
estimatorVersion: 'none',
|
|
204
|
-
reasons: ['an entry in `artifacts` has no resource profile in the catalog']
|
|
205
|
-
}
|
|
223
|
+
result: unknown('an entry in `artifacts` has no resource profile in the catalog')
|
|
206
224
|
};
|
|
207
225
|
}
|
|
226
|
+
const estimated = estimate(candidate, profile, extra, platform, calibration, resources, gpuMode);
|
|
227
|
+
if (estimated.kind === 'estimate' || !floorApplies)
|
|
228
|
+
return { candidate, result: estimated };
|
|
229
|
+
const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
|
|
208
230
|
return {
|
|
209
231
|
candidate,
|
|
210
|
-
result:
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
})
|
|
232
|
+
result: {
|
|
233
|
+
kind: 'floor',
|
|
234
|
+
bytes: floor.bytes,
|
|
235
|
+
reasons: [...estimated.reasons, ...floor.reasons],
|
|
236
|
+
assumptions: [...(estimated.assumptions ?? []), ...floor.assumptions]
|
|
237
|
+
}
|
|
217
238
|
};
|
|
218
239
|
}
|
|
240
|
+
function unknown(reason) {
|
|
241
|
+
return { kind: 'unknown', estimatorVersion: 'none', reasons: [reason] };
|
|
242
|
+
}
|
|
243
|
+
/** Runs the calibrated estimator for a candidate, or says why it cannot. */
|
|
244
|
+
function estimate(candidate, profile, extraArtifactBytes, platform, calibration, resources, gpuMode) {
|
|
245
|
+
if (!calibration) {
|
|
246
|
+
return unknown(platform
|
|
247
|
+
? `no validated calibration for ${platform}`
|
|
248
|
+
: 'the runtime platform is not one this assessment covers');
|
|
249
|
+
}
|
|
250
|
+
// A GPU is present and neither GPU mode engaged, so the only coefficients
|
|
251
|
+
// left were measured with the offload disabled — not how this host runs.
|
|
252
|
+
if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
|
|
253
|
+
return unknown('a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against');
|
|
254
|
+
}
|
|
255
|
+
const estimator = ESTIMATORS[profile.engine];
|
|
256
|
+
if (!estimator) {
|
|
257
|
+
return unknown(`engine '${profile.engine}' has no estimator in this phase`);
|
|
258
|
+
}
|
|
259
|
+
return estimator({
|
|
260
|
+
profile,
|
|
261
|
+
workload: candidate.workload,
|
|
262
|
+
extraArtifactBytes,
|
|
263
|
+
calibration,
|
|
264
|
+
hasGpu: hasGpu(resources)
|
|
265
|
+
});
|
|
266
|
+
}
|
|
219
267
|
/**
|
|
220
268
|
* Sums the artifact bytes of companion constants.
|
|
221
269
|
*
|
|
@@ -545,6 +593,40 @@ function aggregate(estimates, execution) {
|
|
|
545
593
|
}
|
|
546
594
|
return { lower: persistentLower + workingLower, upper: persistentUpper + workingUpper };
|
|
547
595
|
}
|
|
596
|
+
/**
|
|
597
|
+
* The combined lower bound when at least one candidate has only a floor: every
|
|
598
|
+
* estimate contributes its own lower bound, every floor its bytes, aggregated
|
|
599
|
+
* the same way as `aggregate` — a floor has no working peak to add.
|
|
600
|
+
*
|
|
601
|
+
* @returns `undefined` when any candidate has no evidence at all.
|
|
602
|
+
*/
|
|
603
|
+
function aggregateFloor(results, execution) {
|
|
604
|
+
let persistent = 0;
|
|
605
|
+
let working = 0;
|
|
606
|
+
for (const result of results) {
|
|
607
|
+
if (result.kind === 'unknown')
|
|
608
|
+
return undefined;
|
|
609
|
+
if (result.kind === 'floor') {
|
|
610
|
+
persistent += result.bytes;
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
persistent += result.persistent.lower;
|
|
614
|
+
working =
|
|
615
|
+
execution === 'concurrent'
|
|
616
|
+
? working + result.working.lower
|
|
617
|
+
: Math.max(working, result.working.lower);
|
|
618
|
+
}
|
|
619
|
+
return persistent + working;
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* A floor can only refuse: over the largest candidate budget it is
|
|
623
|
+
* `likely-too-large`, and anything else is `unknown`, because the cost above
|
|
624
|
+
* the floor is unmeasured and could be anything.
|
|
625
|
+
*/
|
|
626
|
+
function floorVerdict(floorBytes, candidates) {
|
|
627
|
+
const room = candidates.map((budget) => budget.availableAfterReserveBytes);
|
|
628
|
+
return floorBytes > Math.max(...room) ? 'likely-too-large' : 'unknown';
|
|
629
|
+
}
|
|
548
630
|
function compare(estimate, budget) {
|
|
549
631
|
if (estimate.lower > budget)
|
|
550
632
|
return 'likely-too-large';
|
|
@@ -562,6 +644,24 @@ deviceBudgets, alsoBoundBy) {
|
|
|
562
644
|
reasons: [...result.reasons]
|
|
563
645
|
};
|
|
564
646
|
}
|
|
647
|
+
if (result.kind === 'floor') {
|
|
648
|
+
const verdict = budget ? floorVerdict(result.bytes, deviceBudgets ?? [budget]) : 'unknown';
|
|
649
|
+
return {
|
|
650
|
+
name: candidate.model.name,
|
|
651
|
+
verdict,
|
|
652
|
+
evidence: 'computed-only',
|
|
653
|
+
floorBytes: result.bytes,
|
|
654
|
+
estimatorVersion: FLOOR_VERSION,
|
|
655
|
+
reasons: [
|
|
656
|
+
...result.reasons,
|
|
657
|
+
!budget
|
|
658
|
+
? 'no usable system-memory sample, so this model has no verdict'
|
|
659
|
+
: verdict === 'likely-too-large'
|
|
660
|
+
? 'the computed floor alone exceeds the budget'
|
|
661
|
+
: 'the computed floor is within the budget, but without calibration the cost above it is unbounded, so a fit cannot be claimed'
|
|
662
|
+
]
|
|
663
|
+
};
|
|
664
|
+
}
|
|
565
665
|
const total = {
|
|
566
666
|
lower: result.persistent.lower + result.working.lower,
|
|
567
667
|
upper: result.persistent.upper + result.working.upper
|
|
@@ -569,6 +669,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
569
669
|
return {
|
|
570
670
|
name: candidate.model.name,
|
|
571
671
|
verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
|
|
672
|
+
evidence: 'calibration',
|
|
572
673
|
estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
|
|
573
674
|
estimatorVersion: result.estimatorVersion,
|
|
574
675
|
reasons: budget
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
|
|
2
2
|
import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
|
|
3
3
|
export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
|
|
4
|
+
export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
|
|
4
5
|
/**
|
|
5
6
|
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
6
7
|
* metadata alone.
|