@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
package/dist/plugins/registry.js
CHANGED
|
@@ -5,9 +5,41 @@ import { createAddonLoggerCallback } from '../logging/addon.js';
|
|
|
5
5
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
6
|
import { formatZodError } from '../utils/zod-error.js';
|
|
7
7
|
const plugins = new Map();
|
|
8
|
+
/** Addon logging modules wired through a resolver, keyed by namespace. */
|
|
9
|
+
const lazyAddonLoggers = new Map();
|
|
10
|
+
/** In-flight resolutions, so concurrent loads wire a namespace exactly once. */
|
|
11
|
+
const pendingAddonLoggers = new Map();
|
|
12
|
+
function getLoggingResolver(plugin) {
|
|
13
|
+
const declared = plugin.logging?.module;
|
|
14
|
+
return typeof declared === 'function' ? declared : undefined;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The module a plugin supplied directly, or `undefined` when it supplied a
|
|
18
|
+
* resolver. Resolver-based plugins are therefore invisible to
|
|
19
|
+
* registration-time shape validation, to the shared-module dedupe, and to the
|
|
20
|
+
* eager release sweeps; `ensureAddonLoggerReady` and `lazyAddonLoggers` cover
|
|
21
|
+
* them instead, keyed by namespace rather than held on the plugin.
|
|
22
|
+
*/
|
|
8
23
|
function getLoggingModule(plugin) {
|
|
24
|
+
if (getLoggingResolver(plugin))
|
|
25
|
+
return undefined;
|
|
9
26
|
return plugin.logging?.module;
|
|
10
27
|
}
|
|
28
|
+
/** CommonJS addons resolve through `import()` as a namespace with `default`. */
|
|
29
|
+
function unwrapLoggingModule(resolved) {
|
|
30
|
+
if (resolved && typeof resolved === 'object' && 'default' in resolved) {
|
|
31
|
+
const inner = resolved.default;
|
|
32
|
+
if (inner && typeof inner.setLogger === 'function')
|
|
33
|
+
return inner;
|
|
34
|
+
}
|
|
35
|
+
return resolved;
|
|
36
|
+
}
|
|
37
|
+
function assertLoggingModuleShape(modelType, candidate) {
|
|
38
|
+
if (!candidate || typeof candidate['setLogger'] !== 'function') {
|
|
39
|
+
throw new PluginLoggingInvalidError(modelType, 'logging.module must have a setLogger(callback) function');
|
|
40
|
+
}
|
|
41
|
+
return candidate;
|
|
42
|
+
}
|
|
11
43
|
function findPluginUsingLoggingModule(loggingModule) {
|
|
12
44
|
return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
|
|
13
45
|
}
|
|
@@ -42,12 +74,11 @@ export function registerPlugin(plugin) {
|
|
|
42
74
|
if (pluginProvidingTurboVecIndex) {
|
|
43
75
|
throw new PluginDefinitionInvalidError(plugin.modelType, `plugin "${plugin.modelType}" cannot provide turbovecIndexProvider because plugin "${pluginProvidingTurboVecIndex.modelType}" already provides it`);
|
|
44
76
|
}
|
|
45
|
-
// Validate logging module shape if provided
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
77
|
+
// Validate logging module shape if provided. A resolver is checked when it
|
|
78
|
+
// runs instead: calling it here to inspect its result would load the addon,
|
|
79
|
+
// which is the whole thing registration is meant to avoid.
|
|
80
|
+
if (plugin.logging?.module && !getLoggingResolver(plugin)) {
|
|
81
|
+
assertLoggingModuleShape(plugin.modelType, plugin.logging.module);
|
|
51
82
|
}
|
|
52
83
|
const loggingModule = getLoggingModule(plugin);
|
|
53
84
|
const pluginUsingLoggingModule = loggingModule
|
|
@@ -67,6 +98,91 @@ export function registerPlugins(pluginList) {
|
|
|
67
98
|
registerPlugin(plugin);
|
|
68
99
|
}
|
|
69
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Wires the addon logger of a plugin that supplied a resolver, loading the
|
|
103
|
+
* addon on the way. Call it once a model of this type is actually being
|
|
104
|
+
* created: it is the point where the addon is needed anyway, so a host
|
|
105
|
+
* missing that addon's platform package fails the load it asked for instead
|
|
106
|
+
* of failing every plugin registration at startup.
|
|
107
|
+
*
|
|
108
|
+
* A no-op for plugins that passed their module directly — those are already
|
|
109
|
+
* wired by `registerPlugin` — and for a namespace that is already wired,
|
|
110
|
+
* which is how two plugins over one addon (whisper and Parakeet over ASR)
|
|
111
|
+
* share a single `setLogger` call. Also a no-op when the registry was cleared
|
|
112
|
+
* while the addon was loading, so a shutdown that races a load leaves no
|
|
113
|
+
* logger behind.
|
|
114
|
+
*/
|
|
115
|
+
export async function ensureAddonLoggerReady(plugin) {
|
|
116
|
+
const resolver = getLoggingResolver(plugin);
|
|
117
|
+
const namespace = plugin.logging?.namespace;
|
|
118
|
+
if (!resolver || !namespace)
|
|
119
|
+
return;
|
|
120
|
+
if (lazyAddonLoggers.has(namespace))
|
|
121
|
+
return;
|
|
122
|
+
const pending = pendingAddonLoggers.get(namespace);
|
|
123
|
+
if (pending)
|
|
124
|
+
return pending;
|
|
125
|
+
const wiring = (async () => {
|
|
126
|
+
const resolved = unwrapLoggingModule(await resolver());
|
|
127
|
+
// Loading the addon takes real time, and the registry can be torn down
|
|
128
|
+
// while this is parked: `runCleanup` calls `clearRegistries()`
|
|
129
|
+
// synchronously and only then awaits `unloadAllModels()`, so a sweep can
|
|
130
|
+
// pass between the call above and this line. Wiring now would hand the
|
|
131
|
+
// addon a callback nothing will release — on Expo, a `js_ref_t` leaked
|
|
132
|
+
// into a dying isolate, which the next worklet's first `setLogger` trips
|
|
133
|
+
// over. Nothing has been wired yet, so dropping the resolved module is
|
|
134
|
+
// the whole cleanup.
|
|
135
|
+
if (!isNamespaceClaimed(namespace))
|
|
136
|
+
return;
|
|
137
|
+
const loggingModule = assertLoggingModuleShape(plugin.modelType, resolved);
|
|
138
|
+
for (const [wiredNamespace, wiredModule] of lazyAddonLoggers) {
|
|
139
|
+
if (wiredModule === loggingModule && wiredNamespace !== namespace) {
|
|
140
|
+
throw new PluginLoggingInvalidError(plugin.modelType, 'plugins sharing logging.module must use the same namespace');
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
loggingModule.setLogger(createAddonLoggerCallback(namespace));
|
|
144
|
+
lazyAddonLoggers.set(namespace, loggingModule);
|
|
145
|
+
})();
|
|
146
|
+
pendingAddonLoggers.set(namespace, wiring);
|
|
147
|
+
try {
|
|
148
|
+
await wiring;
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
pendingAddonLoggers.delete(namespace);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** Whether a registered resolver-based plugin still claims this namespace. */
|
|
155
|
+
function isNamespaceClaimed(namespace) {
|
|
156
|
+
return Array.from(plugins.values()).some((candidate) => getLoggingResolver(candidate) && candidate.logging?.namespace === namespace);
|
|
157
|
+
}
|
|
158
|
+
/** Releases a lazily-wired addon logger once no registered plugin claims it. */
|
|
159
|
+
function releaseLazyAddonLogger(namespace, modelType) {
|
|
160
|
+
if (namespace === undefined)
|
|
161
|
+
return;
|
|
162
|
+
const loggingModule = lazyAddonLoggers.get(namespace);
|
|
163
|
+
if (!loggingModule)
|
|
164
|
+
return;
|
|
165
|
+
if (isNamespaceClaimed(namespace))
|
|
166
|
+
return;
|
|
167
|
+
lazyAddonLoggers.delete(namespace);
|
|
168
|
+
releaseLoggerSafely(loggingModule, modelType);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Runs an addon's `releaseLogger`. A failure must not abort a sweep or leave
|
|
172
|
+
* the registry half-cleared for the next caller, but it is reported rather
|
|
173
|
+
* than swallowed: a leaked reference or a live async handle must not pass for
|
|
174
|
+
* a clean teardown. `subject` names what is being released — a model type, or
|
|
175
|
+
* a namespace for a logger that is only keyed by one — and `during` names the
|
|
176
|
+
* sweep, when the release is part of one.
|
|
177
|
+
*/
|
|
178
|
+
function releaseLoggerSafely(loggingModule, subject, during) {
|
|
179
|
+
try {
|
|
180
|
+
loggingModule.releaseLogger?.();
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
getEngineLogger().warn(`[${subject}] releaseLogger failed${during ? ` during ${during}` : ''}: ${error instanceof Error ? error.message : String(error)}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
70
186
|
export function getPlugin(modelType) {
|
|
71
187
|
return plugins.get(modelType);
|
|
72
188
|
}
|
|
@@ -84,10 +200,12 @@ export function unregisterPlugin(modelType) {
|
|
|
84
200
|
if (!plugin)
|
|
85
201
|
return false;
|
|
86
202
|
const loggingModule = getLoggingModule(plugin);
|
|
203
|
+
const resolverNamespace = getLoggingResolver(plugin) ? plugin.logging?.namespace : undefined;
|
|
87
204
|
plugins.delete(modelType);
|
|
88
205
|
if (loggingModule && !findPluginUsingLoggingModule(loggingModule)) {
|
|
89
206
|
loggingModule.releaseLogger?.();
|
|
90
207
|
}
|
|
208
|
+
releaseLazyAddonLogger(resolverNamespace, modelType);
|
|
91
209
|
return true;
|
|
92
210
|
}
|
|
93
211
|
export function getAllPlugins() {
|
|
@@ -104,16 +222,16 @@ export function clearPlugins() {
|
|
|
104
222
|
loggingModules.set(loggingModule, plugin.modelType);
|
|
105
223
|
}
|
|
106
224
|
}
|
|
225
|
+
const lazyModules = new Map(lazyAddonLoggers);
|
|
107
226
|
plugins.clear();
|
|
227
|
+
lazyAddonLoggers.clear();
|
|
108
228
|
for (const [loggingModule, modelType] of loggingModules) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
getEngineLogger().warn(`[${modelType}] releaseLogger failed during clearPlugins: ${error instanceof Error ? error.message : String(error)}`);
|
|
117
|
-
}
|
|
229
|
+
releaseLoggerSafely(loggingModule, modelType, 'clearPlugins');
|
|
230
|
+
}
|
|
231
|
+
// Loggers wired through a resolver are keyed by namespace rather than held
|
|
232
|
+
// on the plugin, so they need the same sweep. They are named by that
|
|
233
|
+
// namespace, which is not a model type — several plugins can share one.
|
|
234
|
+
for (const [namespace, loggingModule] of lazyModules) {
|
|
235
|
+
releaseLoggerSafely(loggingModule, `namespace ${namespace}`, 'clearPlugins');
|
|
118
236
|
}
|
|
119
237
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
import type { VectorIndexBackend } from '../runtime/vector-index-backend.js';
|
|
3
|
+
/**
|
|
4
|
+
* The native index reports only its effective bit width after a load. Three
|
|
5
|
+
* widths map to one storage mode each; 4 bits is either `q4` or
|
|
6
|
+
* `turbovec-q4`, which the snapshot header distinguishes but the addon does
|
|
7
|
+
* not expose yet, so those loads report no storage mode.
|
|
8
|
+
*/
|
|
9
|
+
export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStorage | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* The vector index feature's only backend: the fabric vector index that the
|
|
12
|
+
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
13
|
+
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
14
|
+
* dimension rule (multiple of 8, at most 1024), which the native index
|
|
15
|
+
* enforces itself.
|
|
16
|
+
*/
|
|
17
|
+
export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { getTurboVecIndexProvider } from './registry.js';
|
|
2
|
+
import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
|
|
3
|
+
// The native index pads short result rows with UINT64_MAX.
|
|
4
|
+
const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
|
|
5
|
+
/**
|
|
6
|
+
* The native index reports only its effective bit width after a load. Three
|
|
7
|
+
* widths map to one storage mode each; 4 bits is either `q4` or
|
|
8
|
+
* `turbovec-q4`, which the snapshot header distinguishes but the addon does
|
|
9
|
+
* not expose yet, so those loads report no storage mode.
|
|
10
|
+
*/
|
|
11
|
+
export function storageFromBitWidth(bitWidth) {
|
|
12
|
+
switch (bitWidth) {
|
|
13
|
+
case 32:
|
|
14
|
+
return VectorIndexStorage.F32;
|
|
15
|
+
case 8:
|
|
16
|
+
return VectorIndexStorage.Q8;
|
|
17
|
+
case 2:
|
|
18
|
+
return VectorIndexStorage.TURBOVEC_Q2;
|
|
19
|
+
default:
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function toHitRows(result) {
|
|
24
|
+
const rows = [];
|
|
25
|
+
for (let row = 0; row < result.m; row++) {
|
|
26
|
+
const hits = [];
|
|
27
|
+
for (let slot = 0; slot < result.k; slot++) {
|
|
28
|
+
const offset = row * result.k + slot;
|
|
29
|
+
const id = result.ids[offset];
|
|
30
|
+
if (id === undefined || id === PADDING_ID)
|
|
31
|
+
break;
|
|
32
|
+
hits.push({ id: id.toString(), score: result.scores[offset] ?? 0 });
|
|
33
|
+
}
|
|
34
|
+
rows.push(hits);
|
|
35
|
+
}
|
|
36
|
+
return rows;
|
|
37
|
+
}
|
|
38
|
+
function openIndex(index, storage) {
|
|
39
|
+
// TurboVec precomputes rotation and codebook state on the first search
|
|
40
|
+
// after a change; running it eagerly keeps that cost out of the first
|
|
41
|
+
// query's latency.
|
|
42
|
+
let needsPrepare = true;
|
|
43
|
+
return {
|
|
44
|
+
get dim() {
|
|
45
|
+
return index.dim;
|
|
46
|
+
},
|
|
47
|
+
get length() {
|
|
48
|
+
return index.length;
|
|
49
|
+
},
|
|
50
|
+
storage,
|
|
51
|
+
add(vectors, ids) {
|
|
52
|
+
index.addWithIds(vectors, ids);
|
|
53
|
+
needsPrepare = true;
|
|
54
|
+
},
|
|
55
|
+
search(queries, k) {
|
|
56
|
+
if (needsPrepare) {
|
|
57
|
+
index.prepare();
|
|
58
|
+
needsPrepare = false;
|
|
59
|
+
}
|
|
60
|
+
return toHitRows(index.search(queries, k));
|
|
61
|
+
},
|
|
62
|
+
contains(id) {
|
|
63
|
+
return index.contains(id);
|
|
64
|
+
},
|
|
65
|
+
remove(id) {
|
|
66
|
+
const removed = index.remove(id);
|
|
67
|
+
if (removed)
|
|
68
|
+
needsPrepare = true;
|
|
69
|
+
return removed;
|
|
70
|
+
},
|
|
71
|
+
write(path) {
|
|
72
|
+
index.write(path);
|
|
73
|
+
},
|
|
74
|
+
dispose() {
|
|
75
|
+
index.dispose();
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The vector index feature's only backend: the fabric vector index that the
|
|
81
|
+
* embedding plugin exposes through the `turbovecIndexProvider` capability.
|
|
82
|
+
* It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
|
|
83
|
+
* dimension rule (multiple of 8, at most 1024), which the native index
|
|
84
|
+
* enforces itself.
|
|
85
|
+
*/
|
|
86
|
+
export function getTurboVecBackend() {
|
|
87
|
+
const provider = getTurboVecIndexProvider();
|
|
88
|
+
if (!provider)
|
|
89
|
+
return undefined;
|
|
90
|
+
return {
|
|
91
|
+
create(dim, storage) {
|
|
92
|
+
return openIndex(provider.create({ dim, storage }), storage);
|
|
93
|
+
},
|
|
94
|
+
load(path) {
|
|
95
|
+
const index = provider.load(path);
|
|
96
|
+
return openIndex(index, storageFromBitWidth(index.bitWidth));
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
}
|
package/dist/registry.js
CHANGED
|
@@ -4,6 +4,7 @@ import { handleUnloadModel } from './handlers/unload-model.js';
|
|
|
4
4
|
import { handleLoggingStream } from './handlers/logging-stream.js';
|
|
5
5
|
import { cancelHandler } from './handlers/cancelHandler.js';
|
|
6
6
|
import { handleRag } from './rag/handler.js';
|
|
7
|
+
import { handleVectorIndex } from './handlers/vector-index.js';
|
|
7
8
|
import { handleDeleteCache } from './handlers/delete-cache.js';
|
|
8
9
|
import { handleDownloadAsset } from './handlers/download-asset.js';
|
|
9
10
|
import { handleGetModelInfo } from './handlers/get-model-info.js';
|
|
@@ -74,6 +75,7 @@ export const registry = {
|
|
|
74
75
|
},
|
|
75
76
|
downloadAsset: { type: 'reply', handler: handleDownloadAsset, supportsProgress: true },
|
|
76
77
|
rag: { type: 'reply', handler: handleRag, supportsProgress: ragSupportsProgress },
|
|
78
|
+
vectorIndex: { type: 'reply', handler: handleVectorIndex },
|
|
77
79
|
finetune: { type: 'reply', handler: handleFinetune, supportsProgress: finetuneSupportsProgress },
|
|
78
80
|
// Stream
|
|
79
81
|
completionStream: {
|
|
@@ -94,6 +96,8 @@ export const registry = {
|
|
|
94
96
|
ocrStream: { type: 'stream', pluginOp: true, handler: pluginStream('ocrStream') },
|
|
95
97
|
diffusionStream: { type: 'stream', pluginOp: true, handler: pluginStream('diffusionStream') },
|
|
96
98
|
audioGenStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioGenStream') },
|
|
99
|
+
audioEditStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioEditStream') },
|
|
100
|
+
audioUnderstand: { type: 'stream', pluginOp: true, handler: pluginStream('audioUnderstand') },
|
|
97
101
|
videoStream: { type: 'stream', pluginOp: true, handler: pluginStream('videoStream') },
|
|
98
102
|
upscaleStream: { type: 'stream', pluginOp: true, handler: pluginStream('upscaleStream') },
|
|
99
103
|
worldStepStream: { type: 'stream', pluginOp: true, handler: pluginStream('worldStepStream') },
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { Logger } from '../../../logging/types.js';
|
|
3
|
+
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
|
|
4
|
+
import type { runIsolatedFit } from './run-isolated-fit.js';
|
|
5
|
+
/**
|
|
6
|
+
* `fit` and `does-not-fit` are projections of the load the SDK is about to run,
|
|
7
|
+
* not admission decisions. `@qvac/model-fit` duplicates the loader's policy for
|
|
8
|
+
* this experiment and the real loader neither consumes nor verifies the fitted
|
|
9
|
+
* plan, so neither verdict is denial-grade and no verdict changes the load.
|
|
10
|
+
*
|
|
11
|
+
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
12
|
+
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
13
|
+
* log line.
|
|
14
|
+
*/
|
|
15
|
+
export type AdvisoryFitVerdict = NativeProbeVerdict;
|
|
16
|
+
export type AdvisoryFitOutcome = NativeProbeFit;
|
|
17
|
+
export interface AdvisoryFitInput {
|
|
18
|
+
modelId: string;
|
|
19
|
+
modelType: CanonicalModelType;
|
|
20
|
+
modelPath: string;
|
|
21
|
+
modelConfig: unknown;
|
|
22
|
+
artifacts?: Record<string, string> | undefined;
|
|
23
|
+
isShardedModel: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
27
|
+
* substitute them rather than mocking modules.
|
|
28
|
+
*/
|
|
29
|
+
export interface AdvisoryFitOptions {
|
|
30
|
+
signal?: AbortSignal;
|
|
31
|
+
enabled?: boolean;
|
|
32
|
+
mobile?: boolean;
|
|
33
|
+
timeoutMs?: number;
|
|
34
|
+
runFit?: typeof runIsolatedFit;
|
|
35
|
+
logger?: Logger;
|
|
36
|
+
residentModelBytes?: () => Promise<number>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Runs the advisory llama.cpp fit check for a load that is about to start.
|
|
40
|
+
*
|
|
41
|
+
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
42
|
+
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
43
|
+
* and the caller proceeds with the ordinary load path. This function never
|
|
44
|
+
* throws and never rejects.
|
|
45
|
+
*/
|
|
46
|
+
export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { getEngineLogger } from '../../../logging/index.js';
|
|
2
|
+
import { createLlamaFitRequest } from './create-llama-fit-request.js';
|
|
3
|
+
/**
|
|
4
|
+
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
5
|
+
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
6
|
+
* expiry is `unknown` and the load continues.
|
|
7
|
+
*/
|
|
8
|
+
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
9
|
+
/**
|
|
10
|
+
* `@qvac/model-fit`'s own default margin. Made explicit here because the
|
|
11
|
+
* resident-model reserve below is *added* to it: setting `marginMiB` at all
|
|
12
|
+
* replaces the package default, so the base has to travel with the reserve.
|
|
13
|
+
*/
|
|
14
|
+
const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
15
|
+
/**
|
|
16
|
+
* Names the headroom policy these two numbers add up to: withhold
|
|
17
|
+
* `ADVISORY_FIT_BASE_MARGIN_MIB`, plus the on-disk bytes of every model already
|
|
18
|
+
* resident in this worker.
|
|
19
|
+
*
|
|
20
|
+
* Deliberately *not* `interactive-v1` (withhold 20%, cap 2 GiB desktop / 1 GiB
|
|
21
|
+
* mobile), which `assessModelFit` applies. That policy is applied by the SDK to
|
|
22
|
+
* a budget the SDK computed; here the child owns the budget (total − wired −
|
|
23
|
+
* compressor, measured inside the disposable process) and `marginMiB` is the
|
|
24
|
+
* only lever this side has. Reconciling the two means one entry point owning
|
|
25
|
+
* both bases — see the precedence note on `nativeProbeFitSchema`.
|
|
26
|
+
*/
|
|
27
|
+
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v1';
|
|
28
|
+
const BYTES_PER_MIB = 1024 * 1024;
|
|
29
|
+
const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
|
|
30
|
+
/**
|
|
31
|
+
* On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
|
|
32
|
+
* operator opt-out — the escape hatch for a load-heavy startup path or a
|
|
33
|
+
* runtime where the disposable child cannot spawn. Any other value, including
|
|
34
|
+
* unset, leaves the check on.
|
|
35
|
+
*
|
|
36
|
+
* The worker environment and the mobile runtime flag are imported lazily. Both
|
|
37
|
+
* modules reach Bare-only bindings, and resolving them eagerly would make this
|
|
38
|
+
* orchestration untestable outside a Bare runtime.
|
|
39
|
+
*/
|
|
40
|
+
async function resolveEnabled(explicit) {
|
|
41
|
+
if (explicit !== undefined)
|
|
42
|
+
return explicit;
|
|
43
|
+
const { getValidatedEnv } = await import('../../../runtime/env.js');
|
|
44
|
+
const value = getValidatedEnv().QVAC_ADVISORY_MODEL_FIT;
|
|
45
|
+
return value === undefined || !DISABLED_VALUES.has(value.toLowerCase());
|
|
46
|
+
}
|
|
47
|
+
async function resolveMobile(explicit) {
|
|
48
|
+
if (explicit !== undefined)
|
|
49
|
+
return explicit;
|
|
50
|
+
const { isMobile } = await import('../../../runtime/state.js');
|
|
51
|
+
return isMobile();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Sums the on-disk weight sizes of every model currently registered in this
|
|
55
|
+
* worker. Since `@qvac/model-fit@0.8.0` (qvac-fabric#214) the fit child budgets
|
|
56
|
+
* against system-wide availability (total − wired − compressor), so weights the
|
|
57
|
+
* worker holds *wired* on the GPU are already visible to it. What the child
|
|
58
|
+
* still cannot see is the unwired remainder — mmap'd host-side layers and KV of
|
|
59
|
+
* resident models — which the OS reports as evictable right up until a decode
|
|
60
|
+
* needs it resident. Measured consequence on 0.7.0 (where the child saw a fully
|
|
61
|
+
* idle device): a verdict correct on an idle machine admitted a load that could
|
|
62
|
+
* not decode with another model loaded.
|
|
63
|
+
*
|
|
64
|
+
* Reserving the resident weight bytes through `marginMiB` keeps that footprint
|
|
65
|
+
* in the child's budget. Where those weights are wired this now double-counts
|
|
66
|
+
* and the verdict turns conservative — deliberately so: the measured failure
|
|
67
|
+
* modes punish optimism (loads that cannot decode), not caution.
|
|
68
|
+
*
|
|
69
|
+
* Advisory and fail-open like everything else here: any failure to stat a file
|
|
70
|
+
* contributes zero rather than an error.
|
|
71
|
+
*/
|
|
72
|
+
async function defaultResidentModelBytes() {
|
|
73
|
+
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
74
|
+
import('../../../runtime/model-registry.js'),
|
|
75
|
+
import('bare-fs')
|
|
76
|
+
]);
|
|
77
|
+
let bytes = 0;
|
|
78
|
+
for (const id of getAllModelIds()) {
|
|
79
|
+
const info = getModelInfo(id);
|
|
80
|
+
if (info === null)
|
|
81
|
+
continue;
|
|
82
|
+
try {
|
|
83
|
+
const stats = (await fsPromises.stat(info.path));
|
|
84
|
+
bytes += stats.size;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// Unreadable path: contribute nothing rather than fail the check.
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return bytes;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Also lazy: the desktop supervisor pulls the Bare process launcher, and a
|
|
94
|
+
* disabled check must not load it. Mobile uses in-process `fitParamsAsync` instead
|
|
95
|
+
* of spawn (`bare-runtime/spawn` stays deferred from the mobile pack).
|
|
96
|
+
*/
|
|
97
|
+
async function resolveRunFit(explicit, mobile) {
|
|
98
|
+
if (explicit !== undefined)
|
|
99
|
+
return explicit;
|
|
100
|
+
if (mobile) {
|
|
101
|
+
const { runInProcessFit } = await import('./run-in-process-fit.js');
|
|
102
|
+
return runInProcessFit;
|
|
103
|
+
}
|
|
104
|
+
return (await import('./run-isolated-fit.js')).runIsolatedFit;
|
|
105
|
+
}
|
|
106
|
+
const PROVENANCE = {
|
|
107
|
+
basis: 'native-probe',
|
|
108
|
+
estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
|
|
109
|
+
};
|
|
110
|
+
function unknown(reason, message) {
|
|
111
|
+
return message === undefined
|
|
112
|
+
? { ...PROVENANCE, verdict: 'unknown', reason }
|
|
113
|
+
: { ...PROVENANCE, verdict: 'unknown', reason, message };
|
|
114
|
+
}
|
|
115
|
+
function classify(result) {
|
|
116
|
+
if (result.status === 0) {
|
|
117
|
+
return {
|
|
118
|
+
...PROVENANCE,
|
|
119
|
+
verdict: 'fit',
|
|
120
|
+
reason: result.reason,
|
|
121
|
+
plan: {
|
|
122
|
+
nCtx: result.nCtx,
|
|
123
|
+
nGpuLayers: result.nGpuLayers,
|
|
124
|
+
nGpuDevices: result.nGpuDevices
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
if (result.status === 1) {
|
|
129
|
+
return { ...PROVENANCE, verdict: 'does-not-fit', reason: result.reason };
|
|
130
|
+
}
|
|
131
|
+
// `model-unreadable`, `no-backend-device`, and `unsupported-config` are all
|
|
132
|
+
// absence of evidence, not evidence of insufficiency.
|
|
133
|
+
return unknown(result.reason);
|
|
134
|
+
}
|
|
135
|
+
function report(logger, input, outcome) {
|
|
136
|
+
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
137
|
+
if (outcome.verdict === 'fit') {
|
|
138
|
+
const plan = outcome.plan;
|
|
139
|
+
logger.info(`${prefix} projected to fit (advisory only)${plan === undefined
|
|
140
|
+
? ''
|
|
141
|
+
: ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (outcome.verdict === 'does-not-fit') {
|
|
145
|
+
logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
// `unsupported-load` covers every load this check refuses up front — all
|
|
149
|
+
// non-llama.cpp model types — so at `info` it would tag every whisper/tts/ocr
|
|
150
|
+
// load on every start. `debug` for those; `info` where a fitter ran (or
|
|
151
|
+
// should have) and produced no verdict, because there "why was there no
|
|
152
|
+
// evidence" is the most useful thing this check can report.
|
|
153
|
+
const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
|
|
154
|
+
if (outcome.reason === 'unsupported-load') {
|
|
155
|
+
logger.debug(line);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
logger.info(line);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Runs the advisory llama.cpp fit check for a load that is about to start.
|
|
162
|
+
*
|
|
163
|
+
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
164
|
+
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
165
|
+
* and the caller proceeds with the ordinary load path. This function never
|
|
166
|
+
* throws and never rejects.
|
|
167
|
+
*/
|
|
168
|
+
export async function runAdvisoryFitCheck(input, options = {}) {
|
|
169
|
+
let logger = options.logger;
|
|
170
|
+
try {
|
|
171
|
+
logger ??= getEngineLogger();
|
|
172
|
+
if (!(await resolveEnabled(options.enabled)))
|
|
173
|
+
return unknown('disabled');
|
|
174
|
+
const mobile = await resolveMobile(options.mobile);
|
|
175
|
+
const plan = createLlamaFitRequest({
|
|
176
|
+
modelType: input.modelType,
|
|
177
|
+
modelPath: input.modelPath,
|
|
178
|
+
modelConfig: input.modelConfig,
|
|
179
|
+
artifacts: input.artifacts,
|
|
180
|
+
isShardedModel: input.isShardedModel
|
|
181
|
+
});
|
|
182
|
+
if (!plan.supported) {
|
|
183
|
+
const outcome = unknown('unsupported-load', plan.detail);
|
|
184
|
+
report(logger, input, outcome);
|
|
185
|
+
return outcome;
|
|
186
|
+
}
|
|
187
|
+
const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
|
|
188
|
+
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
189
|
+
const runFit = await resolveRunFit(options.runFit, mobile);
|
|
190
|
+
// Always sent, even with a zero reserve: relying on the addon default for
|
|
191
|
+
// the base margin would leave two sources of truth that match today and
|
|
192
|
+
// diverge silently if the addon default moves.
|
|
193
|
+
const result = await runFit(plan.loadKind, { ...plan.config, marginMiB: ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB }, {
|
|
194
|
+
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
195
|
+
...(options.signal !== undefined && { signal: options.signal })
|
|
196
|
+
});
|
|
197
|
+
const outcome = result.status === 'completed'
|
|
198
|
+
? classify(result.result)
|
|
199
|
+
: unknown(result.reason, result.message);
|
|
200
|
+
report(logger, input, outcome);
|
|
201
|
+
return outcome;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
|
|
205
|
+
try {
|
|
206
|
+
if (logger !== undefined)
|
|
207
|
+
report(logger, input, outcome);
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
// A failing logger must not turn an advisory check into a load failure.
|
|
211
|
+
}
|
|
212
|
+
return outcome;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
|
|
2
|
+
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
3
|
+
export type LlamaFitRequestPlan = {
|
|
4
|
+
supported: true;
|
|
5
|
+
loadKind: LlamaLoadKind;
|
|
6
|
+
config: FitLlamaProcessConfig;
|
|
7
|
+
} | {
|
|
8
|
+
supported: false;
|
|
9
|
+
detail: string;
|
|
10
|
+
};
|
|
11
|
+
export interface CreateLlamaFitRequestParams {
|
|
12
|
+
modelType: CanonicalModelType;
|
|
13
|
+
modelPath: string;
|
|
14
|
+
modelConfig: unknown;
|
|
15
|
+
artifacts?: Record<string, string> | undefined;
|
|
16
|
+
isShardedModel: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Builds a protocol-v2 fit request from the same resolved model config the real
|
|
20
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
21
|
+
*
|
|
22
|
+
* Structural shapes the SDK owns are refused here so the fitter is not asked
|
|
23
|
+
* a question it cannot answer. Value-level policy (device names, symbolic GPU
|
|
24
|
+
* selection, context bounds) stays inside `@qvac/model-fit`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createLlamaFitRequest(params: CreateLlamaFitRequestParams): LlamaFitRequestPlan;
|