@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
import { encodeFitLlamaProcessRequest, FIT_PROCESS_MAX_RESPONSE_BYTES, parseFitProcessResponse, resolveFitProcessRunnerPath } from '@qvac/model-fit/process';
|
|
2
|
+
import env from 'bare-env';
|
|
3
|
+
import spawnBare from 'bare-runtime/spawn';
|
|
4
|
+
import { arch, isAndroid, isBrowser, isIOS, platform } from 'which-runtime';
|
|
5
|
+
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
6
|
+
const TERMINATION_GRACE_MS = 1_000;
|
|
7
|
+
const FINAL_KILL_GRACE_MS = 1_000;
|
|
8
|
+
const DRAIN_GRACE_MS = 1_000;
|
|
9
|
+
const STDERR_TAIL_BYTES = 16 * 1024;
|
|
10
|
+
const FIT_ENVIRONMENT_KEYS = [
|
|
11
|
+
'HOME',
|
|
12
|
+
'PATH',
|
|
13
|
+
'TMPDIR',
|
|
14
|
+
'TMP',
|
|
15
|
+
'TEMP',
|
|
16
|
+
'SystemRoot',
|
|
17
|
+
'WINDIR',
|
|
18
|
+
'LD_LIBRARY_PATH',
|
|
19
|
+
'DYLD_LIBRARY_PATH',
|
|
20
|
+
'VK_ICD_FILENAMES',
|
|
21
|
+
'VK_DRIVER_FILES',
|
|
22
|
+
'CUDA_VISIBLE_DEVICES',
|
|
23
|
+
'HIP_VISIBLE_DEVICES',
|
|
24
|
+
'ROCR_VISIBLE_DEVICES'
|
|
25
|
+
];
|
|
26
|
+
function ignoreLateError() { }
|
|
27
|
+
function allowedEnvironment(environment) {
|
|
28
|
+
const allowed = {};
|
|
29
|
+
for (const key of FIT_ENVIRONMENT_KEYS) {
|
|
30
|
+
const value = environment[key];
|
|
31
|
+
if (value !== undefined)
|
|
32
|
+
allowed[key] = value;
|
|
33
|
+
}
|
|
34
|
+
return allowed;
|
|
35
|
+
}
|
|
36
|
+
function appendTail(current, chunk) {
|
|
37
|
+
const combined = Buffer.concat([current, Buffer.from(chunk)]);
|
|
38
|
+
if (combined.length <= STDERR_TAIL_BYTES)
|
|
39
|
+
return combined;
|
|
40
|
+
let start = combined.length - STDERR_TAIL_BYTES;
|
|
41
|
+
// A byte-offset trim can open mid-codepoint; walk forward past UTF-8
|
|
42
|
+
// continuation bytes (0b10xxxxxx) so toString() never leads with U+FFFD.
|
|
43
|
+
while (start < combined.length && (combined[start] & 0xc0) === 0x80)
|
|
44
|
+
start++;
|
|
45
|
+
return combined.subarray(start);
|
|
46
|
+
}
|
|
47
|
+
function unknown(reason, message, stderrTail) {
|
|
48
|
+
return stderrTail.length === 0
|
|
49
|
+
? { status: 'unknown', reason, message }
|
|
50
|
+
: { status: 'unknown', reason, message, stderrTail: stderrTail.toString() };
|
|
51
|
+
}
|
|
52
|
+
function formatError(error) {
|
|
53
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
54
|
+
}
|
|
55
|
+
function normalizeExitSignal(signal) {
|
|
56
|
+
return signal === null || signal === 0 ? null : String(signal);
|
|
57
|
+
}
|
|
58
|
+
function parseResponse(line, stderrTail) {
|
|
59
|
+
try {
|
|
60
|
+
const response = parseFitProcessResponse(JSON.parse(line));
|
|
61
|
+
if (response.status === 'invocation-error') {
|
|
62
|
+
return unknown('invocation-error', `${response.error.name}: ${response.error.message}`, stderrTail);
|
|
63
|
+
}
|
|
64
|
+
return { status: 'completed', result: response.result };
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
return unknown('invalid-response', formatError(error), stderrTail);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export function runIsolatedFit(loadKind, config, options = {}) {
|
|
71
|
+
const runtime = options.runtime ?? {
|
|
72
|
+
platform,
|
|
73
|
+
arch,
|
|
74
|
+
isAndroid,
|
|
75
|
+
isBrowser,
|
|
76
|
+
isIOS
|
|
77
|
+
};
|
|
78
|
+
if (runtime.isAndroid || runtime.isBrowser || runtime.isIOS) {
|
|
79
|
+
return Promise.resolve({
|
|
80
|
+
status: 'unknown',
|
|
81
|
+
reason: 'unsupported-platform',
|
|
82
|
+
message: `Fit subprocess isolation is unavailable on ${runtime.platform}`
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
const spawnProcess = options.spawnProcess ??
|
|
86
|
+
((context) => spawnBare(context.command, context.options));
|
|
87
|
+
const environment = options.environment ?? env;
|
|
88
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
89
|
+
const terminationGraceMs = options.terminationGraceMs ?? TERMINATION_GRACE_MS;
|
|
90
|
+
const finalKillGraceMs = options.finalKillGraceMs ?? FINAL_KILL_GRACE_MS;
|
|
91
|
+
const drainGraceMs = options.drainGraceMs ?? DRAIN_GRACE_MS;
|
|
92
|
+
return new Promise((resolve) => {
|
|
93
|
+
let child;
|
|
94
|
+
const streams = {
|
|
95
|
+
stdin: undefined,
|
|
96
|
+
stdout: undefined,
|
|
97
|
+
stderr: undefined
|
|
98
|
+
};
|
|
99
|
+
const timers = {
|
|
100
|
+
timeout: undefined,
|
|
101
|
+
forceKill: undefined,
|
|
102
|
+
finalKill: undefined,
|
|
103
|
+
drain: undefined
|
|
104
|
+
};
|
|
105
|
+
let settled = false;
|
|
106
|
+
let destroyingStreams = false;
|
|
107
|
+
let stdout = '';
|
|
108
|
+
let stdoutFailure;
|
|
109
|
+
let stderrTail = Buffer.alloc(0);
|
|
110
|
+
let pendingTermination;
|
|
111
|
+
let exitStatus;
|
|
112
|
+
function safely(action) {
|
|
113
|
+
try {
|
|
114
|
+
action();
|
|
115
|
+
}
|
|
116
|
+
catch { }
|
|
117
|
+
}
|
|
118
|
+
// A destroyed pipe or a reaped child can still emit EPIPE/ERR_STREAM_DESTROYED
|
|
119
|
+
// after the promise settles; without a listener that would take down the host,
|
|
120
|
+
// so the active handler is swapped for an inert one instead of removed.
|
|
121
|
+
function silenceErrors(emitter, active) {
|
|
122
|
+
if (emitter === undefined)
|
|
123
|
+
return;
|
|
124
|
+
safely(() => emitter.on('error', ignoreLateError));
|
|
125
|
+
safely(() => emitter.off('error', active));
|
|
126
|
+
}
|
|
127
|
+
function cleanup() {
|
|
128
|
+
if (timers.timeout !== undefined)
|
|
129
|
+
clearTimeout(timers.timeout);
|
|
130
|
+
if (timers.forceKill !== undefined)
|
|
131
|
+
clearTimeout(timers.forceKill);
|
|
132
|
+
if (timers.finalKill !== undefined)
|
|
133
|
+
clearTimeout(timers.finalKill);
|
|
134
|
+
if (timers.drain !== undefined)
|
|
135
|
+
clearTimeout(timers.drain);
|
|
136
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
137
|
+
safely(() => child?.off('exit', onExit));
|
|
138
|
+
safely(() => child?.off('close', onClose));
|
|
139
|
+
safely(() => streams.stdout?.off('data', onStdoutData));
|
|
140
|
+
safely(() => streams.stderr?.off('data', onStderrData));
|
|
141
|
+
silenceErrors(child, onChildError);
|
|
142
|
+
silenceErrors(streams.stdin, onStdinError);
|
|
143
|
+
silenceErrors(streams.stdout, onStdoutError);
|
|
144
|
+
silenceErrors(streams.stderr, onStderrError);
|
|
145
|
+
}
|
|
146
|
+
function settle(result) {
|
|
147
|
+
if (settled)
|
|
148
|
+
return;
|
|
149
|
+
settled = true;
|
|
150
|
+
cleanup();
|
|
151
|
+
resolve(result);
|
|
152
|
+
}
|
|
153
|
+
function kill(signal) {
|
|
154
|
+
safely(() => {
|
|
155
|
+
child?.kill(signal);
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
function requestTermination(reason, message, crashCanOverride = false) {
|
|
159
|
+
if (settled || exitStatus !== undefined || pendingTermination !== undefined)
|
|
160
|
+
return;
|
|
161
|
+
pendingTermination = { reason, message, crashCanOverride };
|
|
162
|
+
timers.forceKill = setTimeout(() => {
|
|
163
|
+
if (settled || exitStatus !== undefined)
|
|
164
|
+
return;
|
|
165
|
+
kill('SIGKILL');
|
|
166
|
+
if (exitStatus !== undefined)
|
|
167
|
+
return;
|
|
168
|
+
timers.finalKill = setTimeout(() => {
|
|
169
|
+
if (settled || exitStatus !== undefined || pendingTermination === undefined)
|
|
170
|
+
return;
|
|
171
|
+
// Bound liveness when the child cannot be observed as reaped after SIGKILL.
|
|
172
|
+
destroyLocalStreams();
|
|
173
|
+
settle(unknown(pendingTermination.reason, `${pendingTermination.message}; child did not report exit`, stderrTail));
|
|
174
|
+
}, finalKillGraceMs);
|
|
175
|
+
}, terminationGraceMs);
|
|
176
|
+
kill('SIGTERM');
|
|
177
|
+
}
|
|
178
|
+
function destroyLocalStreams() {
|
|
179
|
+
destroyingStreams = true;
|
|
180
|
+
safely(() => streams.stdin?.destroy());
|
|
181
|
+
safely(() => streams.stdout?.destroy());
|
|
182
|
+
safely(() => streams.stderr?.destroy());
|
|
183
|
+
}
|
|
184
|
+
function finalize(code, signal) {
|
|
185
|
+
const newlineIndex = stdout.indexOf('\n');
|
|
186
|
+
const hasTrailingTerminator = newlineIndex === stdout.length - 1;
|
|
187
|
+
const invalidFraming = newlineIndex !== -1 && !hasTrailingTerminator;
|
|
188
|
+
const line = hasTrailingTerminator ? stdout.slice(0, -1) : stdout;
|
|
189
|
+
const response = stdoutFailure === undefined && !invalidFraming && line !== ''
|
|
190
|
+
? parseResponse(line, stderrTail)
|
|
191
|
+
: undefined;
|
|
192
|
+
if (pendingTermination !== undefined && pendingTermination.crashCanOverride === false) {
|
|
193
|
+
settle(unknown(pendingTermination.reason, pendingTermination.message, stderrTail));
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
// A signalled death outranks the runner's own diagnosis: the response was
|
|
197
|
+
// written before whatever killed the child, so it cannot describe the exit.
|
|
198
|
+
if (signal === null &&
|
|
199
|
+
response?.status === 'unknown' &&
|
|
200
|
+
response.reason === 'invocation-error') {
|
|
201
|
+
settle(response);
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
if (code !== 0 || signal !== null) {
|
|
205
|
+
settle(unknown('crashed', `Fit subprocess exited with code ${String(code)} and signal ${String(signal)}`, stderrTail));
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (pendingTermination !== undefined) {
|
|
209
|
+
settle(unknown(pendingTermination.reason, pendingTermination.message, stderrTail));
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (stdoutFailure !== undefined) {
|
|
213
|
+
settle(unknown('invalid-response', stdoutFailure, stderrTail));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
if (invalidFraming || line === '') {
|
|
217
|
+
settle(unknown('invalid-response', invalidFraming
|
|
218
|
+
? 'Fit subprocess returned invalid line framing'
|
|
219
|
+
: `Fit subprocess exited with code ${String(code)} and signal ${String(signal)}`, stderrTail));
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
settle(response ?? parseResponse(line, stderrTail));
|
|
223
|
+
}
|
|
224
|
+
function onAbort() {
|
|
225
|
+
requestTermination('cancelled', 'Fit subprocess was cancelled');
|
|
226
|
+
}
|
|
227
|
+
function onChildError(error) {
|
|
228
|
+
requestTermination('spawn-failed', formatError(error));
|
|
229
|
+
}
|
|
230
|
+
function onExit(code, rawSignal) {
|
|
231
|
+
if (exitStatus !== undefined)
|
|
232
|
+
return;
|
|
233
|
+
const signal = normalizeExitSignal(rawSignal);
|
|
234
|
+
exitStatus = { code, signal };
|
|
235
|
+
if (timers.timeout !== undefined) {
|
|
236
|
+
clearTimeout(timers.timeout);
|
|
237
|
+
timers.timeout = undefined;
|
|
238
|
+
}
|
|
239
|
+
if (timers.forceKill !== undefined) {
|
|
240
|
+
clearTimeout(timers.forceKill);
|
|
241
|
+
timers.forceKill = undefined;
|
|
242
|
+
}
|
|
243
|
+
if (timers.finalKill !== undefined) {
|
|
244
|
+
clearTimeout(timers.finalKill);
|
|
245
|
+
timers.finalKill = undefined;
|
|
246
|
+
}
|
|
247
|
+
timers.drain = setTimeout(() => {
|
|
248
|
+
destroyLocalStreams();
|
|
249
|
+
finalize(code, signal);
|
|
250
|
+
}, drainGraceMs);
|
|
251
|
+
}
|
|
252
|
+
function onClose(code, rawSignal) {
|
|
253
|
+
const status = exitStatus ?? { code, signal: normalizeExitSignal(rawSignal) };
|
|
254
|
+
finalize(status.code, status.signal);
|
|
255
|
+
}
|
|
256
|
+
function onStdoutData(chunk) {
|
|
257
|
+
if (settled || pendingTermination !== undefined || destroyingStreams) {
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
if (Buffer.byteLength(stdout) + Buffer.byteLength(chunk) > FIT_PROCESS_MAX_RESPONSE_BYTES) {
|
|
261
|
+
if (exitStatus !== undefined) {
|
|
262
|
+
stdoutFailure = 'Fit subprocess response exceeds 1 MiB';
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
requestTermination('invalid-response', 'Fit subprocess response exceeds 1 MiB');
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
stdout += chunk;
|
|
269
|
+
const newlineIndex = stdout.indexOf('\n');
|
|
270
|
+
if (newlineIndex !== -1 && newlineIndex !== stdout.length - 1) {
|
|
271
|
+
if (exitStatus !== undefined) {
|
|
272
|
+
stdoutFailure = 'Fit subprocess returned invalid line framing';
|
|
273
|
+
}
|
|
274
|
+
else {
|
|
275
|
+
requestTermination('invalid-response', 'Fit subprocess returned invalid line framing');
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
function onStderrData(chunk) {
|
|
280
|
+
if (settled || destroyingStreams)
|
|
281
|
+
return;
|
|
282
|
+
stderrTail = appendTail(stderrTail, chunk);
|
|
283
|
+
}
|
|
284
|
+
function onStdinError(error) {
|
|
285
|
+
if (destroyingStreams)
|
|
286
|
+
return;
|
|
287
|
+
requestTermination('invalid-response', `Fit subprocess stdin failed: ${error.message}`, true);
|
|
288
|
+
}
|
|
289
|
+
function onStdoutError(error) {
|
|
290
|
+
if (destroyingStreams)
|
|
291
|
+
return;
|
|
292
|
+
requestTermination('invalid-response', `Fit subprocess stdout failed: ${error.message}`);
|
|
293
|
+
}
|
|
294
|
+
function onStderrError(error) {
|
|
295
|
+
if (destroyingStreams)
|
|
296
|
+
return;
|
|
297
|
+
requestTermination('invalid-response', `Fit subprocess stderr failed: ${error.message}`);
|
|
298
|
+
}
|
|
299
|
+
try {
|
|
300
|
+
const stdio = runtime.platform === 'win32'
|
|
301
|
+
? ['overlapped', 'overlapped', 'overlapped']
|
|
302
|
+
: ['pipe', 'pipe', 'pipe'];
|
|
303
|
+
child = spawnProcess({
|
|
304
|
+
command: 'bare',
|
|
305
|
+
options: {
|
|
306
|
+
args: [
|
|
307
|
+
options.runnerPath ?? resolveFitProcessRunnerPath(),
|
|
308
|
+
...(options.runnerArgs ?? [])
|
|
309
|
+
],
|
|
310
|
+
platform: runtime.platform,
|
|
311
|
+
arch: runtime.arch,
|
|
312
|
+
stdio,
|
|
313
|
+
env: allowedEnvironment(environment)
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
catch (error) {
|
|
318
|
+
settle(unknown('spawn-failed', formatError(error), stderrTail));
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
child.on('error', onChildError);
|
|
322
|
+
child.on('exit', onExit);
|
|
323
|
+
child.on('close', onClose);
|
|
324
|
+
const stdin = child.stdin;
|
|
325
|
+
const stdoutStream = child.stdout;
|
|
326
|
+
const stderrStream = child.stderr;
|
|
327
|
+
streams.stdin = stdin ?? undefined;
|
|
328
|
+
streams.stdout = stdoutStream ?? undefined;
|
|
329
|
+
streams.stderr = stderrStream ?? undefined;
|
|
330
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
331
|
+
timers.timeout = setTimeout(() => {
|
|
332
|
+
requestTermination('timeout', `Fit subprocess exceeded ${timeoutMs}ms`);
|
|
333
|
+
}, timeoutMs);
|
|
334
|
+
try {
|
|
335
|
+
streams.stdin?.on('error', onStdinError);
|
|
336
|
+
streams.stdout?.on('data', onStdoutData);
|
|
337
|
+
streams.stdout?.on('error', onStdoutError);
|
|
338
|
+
streams.stderr?.on('data', onStderrData);
|
|
339
|
+
streams.stderr?.on('error', onStderrError);
|
|
340
|
+
}
|
|
341
|
+
catch (error) {
|
|
342
|
+
requestTermination('invalid-response', `Fit subprocess stream setup failed: ${formatError(error)}`);
|
|
343
|
+
destroyLocalStreams();
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
if (stdin === null || stdoutStream === null || stderrStream === null) {
|
|
347
|
+
requestTermination('spawn-failed', 'Fit subprocess stdio pipes are unavailable');
|
|
348
|
+
destroyLocalStreams();
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
try {
|
|
352
|
+
stdoutStream.setEncoding('utf8');
|
|
353
|
+
stderrStream.setEncoding('utf8');
|
|
354
|
+
}
|
|
355
|
+
catch (error) {
|
|
356
|
+
requestTermination('invalid-response', `Fit subprocess stream setup failed: ${formatError(error)}`);
|
|
357
|
+
destroyLocalStreams();
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
if (options.signal?.aborted === true) {
|
|
361
|
+
onAbort();
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
try {
|
|
365
|
+
stdin.end(encodeFitLlamaProcessRequest(loadKind, config));
|
|
366
|
+
}
|
|
367
|
+
catch (error) {
|
|
368
|
+
requestTermination('invalid-response', `Fit subprocess request write failed: ${formatError(error)}`);
|
|
369
|
+
destroyLocalStreams();
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
}
|
package/dist/runtime/env.d.ts
CHANGED
|
@@ -7,10 +7,12 @@ export declare function initEnv(): void;
|
|
|
7
7
|
*/
|
|
8
8
|
export declare function getEnv(): {
|
|
9
9
|
HOME_DIR: string;
|
|
10
|
+
QVAC_ADVISORY_MODEL_FIT?: string | undefined;
|
|
10
11
|
};
|
|
11
12
|
/**
|
|
12
13
|
* Get the validated env config. Must call initEnv() first.
|
|
13
14
|
*/
|
|
14
15
|
export declare function getValidatedEnv(): {
|
|
15
16
|
HOME_DIR: string;
|
|
17
|
+
QVAC_ADVISORY_MODEL_FIT?: string | undefined;
|
|
16
18
|
};
|
package/dist/runtime/env.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import env from 'bare-env';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
const envSchema = z.object({
|
|
4
|
-
HOME_DIR: z.string()
|
|
4
|
+
HOME_DIR: z.string(),
|
|
5
|
+
/**
|
|
6
|
+
* Operator opt-out for the advisory llama.cpp fit check (on by default).
|
|
7
|
+
* Free-form rather than an enum so a typo cannot fail engine startup; only
|
|
8
|
+
* `0`/`false`/`off`/`no` disable it.
|
|
9
|
+
*/
|
|
10
|
+
QVAC_ADVISORY_MODEL_FIT: z.string().optional()
|
|
5
11
|
});
|
|
6
12
|
let validatedEnv = null;
|
|
7
13
|
/**
|
|
@@ -12,7 +18,8 @@ export function initEnv() {
|
|
|
12
18
|
// Snap's HOME can be revision-scoped; SNAP_USER_COMMON is stable.
|
|
13
19
|
env['SNAP_USER_COMMON'] ?? env['HOME'] ?? env['USERPROFILE'] ?? '/tmp';
|
|
14
20
|
let envConfig = {
|
|
15
|
-
HOME_DIR: defaultHomeDir
|
|
21
|
+
HOME_DIR: defaultHomeDir,
|
|
22
|
+
QVAC_ADVISORY_MODEL_FIT: env['QVAC_ADVISORY_MODEL_FIT']
|
|
16
23
|
};
|
|
17
24
|
const isBareKit = typeof globalThis.BareKit !== 'undefined';
|
|
18
25
|
if (isBareKit && Bare.argv[0]) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { initEnv } from './env.js';
|
|
2
2
|
import { closeAllRagInstances } from '../rag/index.js';
|
|
3
|
+
import { disposeAllVectorIndexes } from './vector-index-registry.js';
|
|
3
4
|
import { cleanupDownloads } from '../handlers/load-model/download-manager.js';
|
|
4
5
|
import { unloadAllModels } from './model-registry.js';
|
|
5
6
|
import { closeRegistryClient } from './registry-client.js';
|
|
@@ -57,6 +58,7 @@ async function runCleanup() {
|
|
|
57
58
|
cleanupRan = true;
|
|
58
59
|
destroyResourceCollector();
|
|
59
60
|
clearRegistries();
|
|
61
|
+
disposeAllVectorIndexes();
|
|
60
62
|
await Promise.allSettled([
|
|
61
63
|
closeAllRagInstances(),
|
|
62
64
|
cleanupDownloads(),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CanonicalModelType } from '../schemas/index.js';
|
|
1
|
+
import type { CanonicalModelType, NativeProbeFit } from '../schemas/index.js';
|
|
2
2
|
interface AddonInterface {
|
|
3
3
|
cancel(jobId?: string): Promise<void>;
|
|
4
4
|
}
|
|
@@ -26,6 +26,7 @@ interface LocalOptions {
|
|
|
26
26
|
config: unknown;
|
|
27
27
|
modelType: CanonicalModelType;
|
|
28
28
|
name?: string | undefined;
|
|
29
|
+
fitProbe?: NativeProbeFit | undefined;
|
|
29
30
|
}
|
|
30
31
|
export type ModelEntry = {
|
|
31
32
|
id: string;
|
|
@@ -37,6 +38,7 @@ export declare function registerModel(id: string, options: {
|
|
|
37
38
|
config: unknown;
|
|
38
39
|
modelType: CanonicalModelType;
|
|
39
40
|
name?: string | undefined;
|
|
41
|
+
fitProbe?: NativeProbeFit | undefined;
|
|
40
42
|
}): void;
|
|
41
43
|
export declare function getModelEntry(id: string): ModelEntry | null;
|
|
42
44
|
export declare function getModel(id: string): AnyModel;
|
|
@@ -15,7 +15,8 @@ export function registerModel(id, options) {
|
|
|
15
15
|
loadedAt: new Date(),
|
|
16
16
|
config: options.config,
|
|
17
17
|
modelType: options.modelType,
|
|
18
|
-
name: options.name
|
|
18
|
+
name: options.name,
|
|
19
|
+
fitProbe: options.fitProbe
|
|
19
20
|
}
|
|
20
21
|
});
|
|
21
22
|
const nameStr = options.name ? ` (${options.name})` : '';
|
|
@@ -65,6 +65,23 @@ function installDefaultPolicies(r) {
|
|
|
65
65
|
onOverflow: 'queue',
|
|
66
66
|
maxQueueDepthPerModel: 64
|
|
67
67
|
});
|
|
68
|
+
// @qvac/tts-ggml is constructed with `exclusiveRun: true`, so it already
|
|
69
|
+
// serialises synthesis per model — but it queues internally, where the
|
|
70
|
+
// registry cannot see it. Mirroring the limit here keeps the registry
|
|
71
|
+
// authoritative, which is what model-scoped cancel needs: exactly one
|
|
72
|
+
// registered request owns the addon at a time, so cancelling a queued run
|
|
73
|
+
// aborts it before it starts instead of interrupting the run in flight.
|
|
74
|
+
// Deeper than the other lanes on purpose: the addon's own queue was
|
|
75
|
+
// unbounded, and a caller fanning a paragraph out sentence-by-sentence
|
|
76
|
+
// (`Promise.all` over a few dozen `textToSpeech` calls) is a realistic
|
|
77
|
+
// pattern that must keep working. Beyond this the call rejects with a typed
|
|
78
|
+
// RequestRejectedByPolicyError instead of piling up.
|
|
79
|
+
r.policy({
|
|
80
|
+
kind: 'tts',
|
|
81
|
+
maxConcurrentPerModel: 1,
|
|
82
|
+
onOverflow: 'queue',
|
|
83
|
+
maxQueueDepthPerModel: 256
|
|
84
|
+
});
|
|
68
85
|
// An ABot-World session runs one job at a time: the addon rejects a second
|
|
69
86
|
// step while a block is still streaming, and scene creation shares the same
|
|
70
87
|
// lane. Reject rather than queue, unlike every other policy here, because a
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { VectorIndexHit, VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* An open index as the vector index feature sees it. Row-major vectors and
|
|
4
|
+
* unsigned 64-bit ids go in; finished hit rows come out. Anything specific
|
|
5
|
+
* to a native engine (warm-up, result padding, how the storage mode is
|
|
6
|
+
* reported) is handled by the backend that implements this, not by callers.
|
|
7
|
+
*/
|
|
8
|
+
export interface OpenVectorIndex {
|
|
9
|
+
readonly dim: number;
|
|
10
|
+
readonly length: number;
|
|
11
|
+
readonly storage: VectorIndexStorage | undefined;
|
|
12
|
+
/** Adds `ids.length` vectors of `dim` components each, atomically. */
|
|
13
|
+
add(vectors: Float32Array, ids: BigUint64Array): void;
|
|
14
|
+
/** One hit list per query row, ordered by descending score; never padded. */
|
|
15
|
+
search(queries: Float32Array, k: number): VectorIndexHit[][];
|
|
16
|
+
contains(id: bigint): boolean;
|
|
17
|
+
/** Returns whether the id was present. */
|
|
18
|
+
remove(id: bigint): boolean;
|
|
19
|
+
write(path: string): void;
|
|
20
|
+
dispose(): void;
|
|
21
|
+
}
|
|
22
|
+
export interface VectorIndexBackend {
|
|
23
|
+
create(dim: number, storage: VectorIndexStorage): OpenVectorIndex;
|
|
24
|
+
load(path: string): OpenVectorIndex;
|
|
25
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { VectorIdWire, VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Snapshot paths are used as given when absolute. A relative path resolves
|
|
4
|
+
* against the QVAC data directory (the parent of the configured cache
|
|
5
|
+
* directory), the same root that holds the RAG workspace stores, so the
|
|
6
|
+
* same relative path works on every platform.
|
|
7
|
+
*/
|
|
8
|
+
export declare function resolveVectorIndexPath(snapshotPath: string): string;
|
|
9
|
+
export declare function createVectorIndex(params: {
|
|
10
|
+
dim: number;
|
|
11
|
+
storage: VectorIndexStorage;
|
|
12
|
+
}): {
|
|
13
|
+
indexId: string;
|
|
14
|
+
dim: number;
|
|
15
|
+
storage: VectorIndexStorage;
|
|
16
|
+
length: number;
|
|
17
|
+
};
|
|
18
|
+
export declare function loadVectorIndex(params: {
|
|
19
|
+
path: string;
|
|
20
|
+
}): {
|
|
21
|
+
storage?: VectorIndexStorage;
|
|
22
|
+
indexId: string;
|
|
23
|
+
dim: number;
|
|
24
|
+
length: number;
|
|
25
|
+
};
|
|
26
|
+
export declare function addVectors(params: {
|
|
27
|
+
indexId: string;
|
|
28
|
+
ids: VectorIdWire[];
|
|
29
|
+
vectors: number[][];
|
|
30
|
+
}): {
|
|
31
|
+
length: number;
|
|
32
|
+
};
|
|
33
|
+
export declare function searchVectors(params: {
|
|
34
|
+
indexId: string;
|
|
35
|
+
queries: number[][];
|
|
36
|
+
k: number;
|
|
37
|
+
}): {
|
|
38
|
+
results: {
|
|
39
|
+
id: string;
|
|
40
|
+
score: number;
|
|
41
|
+
}[][];
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Removes ids one at a time, which is the only shape the backend offers. A
|
|
45
|
+
* backend throw here means the handle itself is unusable, so the batch stops
|
|
46
|
+
* rather than trying the rest — but ids before it are already gone, so the
|
|
47
|
+
* error names how many were applied. `length` is only reported on success,
|
|
48
|
+
* which is why the handle documents its own `length` as of the last call that
|
|
49
|
+
* reported one.
|
|
50
|
+
*/
|
|
51
|
+
export declare function removeVectors(params: {
|
|
52
|
+
indexId: string;
|
|
53
|
+
ids: VectorIdWire[];
|
|
54
|
+
}): {
|
|
55
|
+
removed: boolean[];
|
|
56
|
+
length: number;
|
|
57
|
+
};
|
|
58
|
+
export declare function containsVectors(params: {
|
|
59
|
+
indexId: string;
|
|
60
|
+
ids: VectorIdWire[];
|
|
61
|
+
}): {
|
|
62
|
+
present: boolean[];
|
|
63
|
+
};
|
|
64
|
+
export declare function writeVectorIndex(params: {
|
|
65
|
+
indexId: string;
|
|
66
|
+
path: string;
|
|
67
|
+
}): Promise<{
|
|
68
|
+
path: string;
|
|
69
|
+
}>;
|
|
70
|
+
export declare function disposeVectorIndex(params: {
|
|
71
|
+
indexId: string;
|
|
72
|
+
}): {
|
|
73
|
+
disposed: boolean;
|
|
74
|
+
};
|
|
75
|
+
export declare function getOpenVectorIndexCount(): number;
|
|
76
|
+
export declare function disposeAllVectorIndexes(): void;
|