@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
package/dist/api/transcribe.js
CHANGED
|
@@ -144,16 +144,47 @@ export async function createTranscribeStreamSession(params, options, process, se
|
|
|
144
144
|
statsSettled = true;
|
|
145
145
|
rejectStatsPromise(error);
|
|
146
146
|
}
|
|
147
|
-
|
|
148
|
-
|
|
147
|
+
// Settled from the same terminal frame as `stats`, and on the same paths:
|
|
148
|
+
// resolved empty when the stream closes without one, rejected on error.
|
|
149
|
+
let diagnosticsSettled = false;
|
|
150
|
+
let resolveDiagnosticsPromise = () => { };
|
|
151
|
+
let rejectDiagnosticsPromise = () => { };
|
|
152
|
+
const diagnostics = new Promise((resolve, reject) => {
|
|
153
|
+
resolveDiagnosticsPromise = resolve;
|
|
154
|
+
rejectDiagnosticsPromise = reject;
|
|
155
|
+
});
|
|
156
|
+
diagnostics.catch(() => { });
|
|
157
|
+
function resolveDiagnostics(value) {
|
|
158
|
+
if (diagnosticsSettled)
|
|
159
|
+
return;
|
|
160
|
+
diagnosticsSettled = true;
|
|
161
|
+
resolveDiagnosticsPromise(value);
|
|
162
|
+
}
|
|
163
|
+
function rejectDiagnostics(error) {
|
|
164
|
+
if (diagnosticsSettled)
|
|
165
|
+
return;
|
|
166
|
+
diagnosticsSettled = true;
|
|
167
|
+
rejectDiagnosticsPromise(error);
|
|
168
|
+
}
|
|
169
|
+
function resolveTerminal(statsValue, diagnosticsValue) {
|
|
170
|
+
resolveStats(statsValue);
|
|
171
|
+
resolveDiagnostics(diagnosticsValue);
|
|
172
|
+
}
|
|
173
|
+
function resolveEmptyTerminal() {
|
|
174
|
+
resolveTerminal(undefined, undefined);
|
|
175
|
+
}
|
|
176
|
+
function rejectTerminal(error) {
|
|
177
|
+
rejectStats(error);
|
|
178
|
+
rejectDiagnostics(error);
|
|
149
179
|
}
|
|
150
180
|
function processResponse(line) {
|
|
151
|
-
return process(line,
|
|
181
|
+
return process(line, resolveTerminal);
|
|
152
182
|
}
|
|
153
|
-
const responses = parseLines(responseStream, processResponse,
|
|
183
|
+
const responses = parseLines(responseStream, processResponse, resolveEmptyTerminal, rejectTerminal);
|
|
154
184
|
let consumed = false;
|
|
155
185
|
return {
|
|
156
186
|
stats,
|
|
187
|
+
diagnostics,
|
|
157
188
|
write(audioChunk) {
|
|
158
189
|
requestStream.write(audioChunk);
|
|
159
190
|
},
|
|
@@ -161,7 +192,7 @@ export async function createTranscribeStreamSession(params, options, process, se
|
|
|
161
192
|
requestStream.end();
|
|
162
193
|
},
|
|
163
194
|
destroy() {
|
|
164
|
-
|
|
195
|
+
resolveEmptyTerminal();
|
|
165
196
|
requestStream.destroy();
|
|
166
197
|
responseStream.destroy();
|
|
167
198
|
},
|
|
@@ -182,7 +213,7 @@ function transcribeStreamDuplexMetadata(params, options) {
|
|
|
182
213
|
}
|
|
183
214
|
function transcribeStreamDuplexConversation(params, options) {
|
|
184
215
|
const wantsMetadata = params.metadata === true;
|
|
185
|
-
return createTranscribeStreamSession(params, options, (line,
|
|
216
|
+
return createTranscribeStreamSession(params, options, (line, onTerminal) => processLineConversation(line, wantsMetadata, onTerminal), 'TranscribeStreamConversationSession');
|
|
186
217
|
}
|
|
187
218
|
/**
|
|
188
219
|
* Line-delimited parser: reads newline-separated frames from a duplex
|
|
@@ -241,31 +272,32 @@ function parseResponseLine(line) {
|
|
|
241
272
|
* `undefined` for frames the caller should skip, or the value extracted from
|
|
242
273
|
* the frame.
|
|
243
274
|
*/
|
|
244
|
-
function processWith(line, extract,
|
|
275
|
+
function processWith(line, extract, onTerminal) {
|
|
245
276
|
const response = parseResponseLine(line);
|
|
246
277
|
if (response === null)
|
|
247
278
|
return undefined;
|
|
248
279
|
if (response.error)
|
|
249
280
|
throw new TranscriptionFailedError(response.error);
|
|
250
281
|
if (response.done) {
|
|
251
|
-
|
|
282
|
+
onTerminal(response.stats, response.diagnostics);
|
|
252
283
|
return null;
|
|
253
284
|
}
|
|
254
285
|
return extract(response);
|
|
255
286
|
}
|
|
256
|
-
export function processLine(line,
|
|
257
|
-
return processWith(line, (response) => (response.text?.trim() ? response.text : undefined),
|
|
287
|
+
export function processLine(line, onTerminal) {
|
|
288
|
+
return processWith(line, (response) => (response.text?.trim() ? response.text : undefined), onTerminal);
|
|
258
289
|
}
|
|
259
|
-
function processLineMetadata(line,
|
|
260
|
-
return processWith(line, (response) => response.segment,
|
|
290
|
+
function processLineMetadata(line, onTerminal) {
|
|
291
|
+
return processWith(line, (response) => response.segment, onTerminal);
|
|
261
292
|
}
|
|
262
|
-
function processLineConversation(line, wantsMetadata,
|
|
293
|
+
export function processLineConversation(line, wantsMetadata, onTerminal) {
|
|
263
294
|
return processWith(line, (response) => {
|
|
264
295
|
if (response.vad) {
|
|
265
296
|
return {
|
|
266
297
|
type: 'vad',
|
|
267
298
|
speaking: response.vad.speaking,
|
|
268
|
-
probability: response.vad.probability
|
|
299
|
+
probability: response.vad.probability,
|
|
300
|
+
...(response.vad.source && { source: response.vad.source })
|
|
269
301
|
};
|
|
270
302
|
}
|
|
271
303
|
if (response.endOfTurn) {
|
|
@@ -291,5 +323,5 @@ function processLineConversation(line, wantsMetadata, onStats) {
|
|
|
291
323
|
return { type: 'text', text: response.text };
|
|
292
324
|
}
|
|
293
325
|
return undefined;
|
|
294
|
-
},
|
|
326
|
+
}, onTerminal);
|
|
295
327
|
}
|
|
@@ -6,9 +6,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
6
6
|
init_image?: string;
|
|
7
7
|
control_frames?: string[];
|
|
8
8
|
modelId: string;
|
|
9
|
+
prompt: string;
|
|
9
10
|
seed?: number | undefined;
|
|
10
11
|
vae_tiling?: boolean | undefined;
|
|
11
|
-
prompt: string;
|
|
12
12
|
negative_prompt?: string | undefined;
|
|
13
13
|
width?: number | undefined;
|
|
14
14
|
height?: number | undefined;
|
|
@@ -47,9 +47,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
47
47
|
init_image?: string;
|
|
48
48
|
control_frames?: string[];
|
|
49
49
|
modelId: string;
|
|
50
|
+
prompt: string;
|
|
50
51
|
seed?: number | undefined;
|
|
51
52
|
vae_tiling?: boolean | undefined;
|
|
52
|
-
prompt: string;
|
|
53
53
|
negative_prompt?: string | undefined;
|
|
54
54
|
width?: number | undefined;
|
|
55
55
|
height?: number | undefined;
|
|
@@ -88,9 +88,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
88
88
|
init_image?: string;
|
|
89
89
|
control_frames?: string[];
|
|
90
90
|
modelId: string;
|
|
91
|
+
prompt: string;
|
|
91
92
|
seed?: number | undefined;
|
|
92
93
|
vae_tiling?: boolean | undefined;
|
|
93
|
-
prompt: string;
|
|
94
94
|
negative_prompt?: string | undefined;
|
|
95
95
|
width?: number | undefined;
|
|
96
96
|
height?: number | undefined;
|
|
@@ -129,9 +129,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
129
129
|
init_image?: string;
|
|
130
130
|
control_frames?: string[];
|
|
131
131
|
modelId: string;
|
|
132
|
+
prompt: string;
|
|
132
133
|
seed?: number | undefined;
|
|
133
134
|
vae_tiling?: boolean | undefined;
|
|
134
|
-
prompt: string;
|
|
135
135
|
negative_prompt?: string | undefined;
|
|
136
136
|
width?: number | undefined;
|
|
137
137
|
height?: number | undefined;
|
|
@@ -170,9 +170,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
170
170
|
init_image?: string;
|
|
171
171
|
control_frames?: string[];
|
|
172
172
|
modelId: string;
|
|
173
|
+
prompt: string;
|
|
173
174
|
seed?: number | undefined;
|
|
174
175
|
vae_tiling?: boolean | undefined;
|
|
175
|
-
prompt: string;
|
|
176
176
|
negative_prompt?: string | undefined;
|
|
177
177
|
width?: number | undefined;
|
|
178
178
|
height?: number | undefined;
|
package/dist/dispatch.js
CHANGED
|
@@ -9,6 +9,7 @@ import { assertLifecycleAllowed } from './runtime/runtime-lifecycle.js';
|
|
|
9
9
|
import { resolveModelConfig, setConfig, setRuntimeContext, isConfigSet } from './runtime/state.js';
|
|
10
10
|
import { initialize, close as closeEngine } from './runtime/lifecycle.js';
|
|
11
11
|
import { getAllPlugins } from './plugins/index.js';
|
|
12
|
+
import { hasRpcServerProvider } from './rpc/provider.js';
|
|
12
13
|
import { resolveConfig } from './config/resolve-config.js';
|
|
13
14
|
import { setGlobalLogLevel, setGlobalConsoleOutput, getAppLogger } from './logging/index.js';
|
|
14
15
|
import { profileReplyHandler, profileStreamHandler } from './profiling/index.js';
|
|
@@ -24,7 +25,7 @@ let readyPromise = null;
|
|
|
24
25
|
// does not mark readiness.
|
|
25
26
|
let generation = 0;
|
|
26
27
|
function ensurePluginsRegistered() {
|
|
27
|
-
if (getAllPlugins().length === 0) {
|
|
28
|
+
if (getAllPlugins().length === 0 && !hasRpcServerProvider()) {
|
|
28
29
|
throw new PluginsNotRegisteredError();
|
|
29
30
|
}
|
|
30
31
|
}
|
|
@@ -56,14 +57,15 @@ async function initializeConfig() {
|
|
|
56
57
|
}
|
|
57
58
|
async function performReady(gen) {
|
|
58
59
|
initialize();
|
|
59
|
-
ensurePluginsRegistered();
|
|
60
60
|
await initializeConfig();
|
|
61
61
|
// Mark ready only while this generation is current — a `close()` during the
|
|
62
62
|
// await bumps it past this now-superseded init.
|
|
63
63
|
if (gen === generation)
|
|
64
64
|
ready = true;
|
|
65
65
|
}
|
|
66
|
-
async function ensureReady() {
|
|
66
|
+
async function ensureReady(allowWithoutPlugins = false) {
|
|
67
|
+
if (!allowWithoutPlugins)
|
|
68
|
+
ensurePluginsRegistered();
|
|
67
69
|
if (ready)
|
|
68
70
|
return;
|
|
69
71
|
// A single shared promise serializes concurrent first calls: `ready` only
|
|
@@ -88,11 +90,19 @@ function getHandlerEntry(type) {
|
|
|
88
90
|
return entry;
|
|
89
91
|
}
|
|
90
92
|
/**
|
|
91
|
-
*
|
|
92
|
-
* handler runs, matching the priority user config >
|
|
93
|
-
* defaults
|
|
93
|
+
* Fills a request that describes a load with device and schema config
|
|
94
|
+
* defaults before the handler runs, matching the priority user config >
|
|
95
|
+
* device defaults > schema defaults.
|
|
96
|
+
*
|
|
97
|
+
* `loadModel` and every `assessModelFit` candidate describe the same load, so
|
|
98
|
+
* both are filled the same way: the fitter is asked about the settings a real
|
|
99
|
+
* load resolves, and refuses one carrying no `device` at all. Every other
|
|
100
|
+
* request type passes through untouched.
|
|
94
101
|
*/
|
|
95
102
|
function applyDeviceDefaults(request) {
|
|
103
|
+
if (request.type === 'assessModelFit' && 'models' in request) {
|
|
104
|
+
return applyCandidateDeviceDefaults(request);
|
|
105
|
+
}
|
|
96
106
|
if (request.type !== 'loadModel' || !('modelSrc' in request))
|
|
97
107
|
return request;
|
|
98
108
|
let canonicalType;
|
|
@@ -105,6 +115,25 @@ function applyDeviceDefaults(request) {
|
|
|
105
115
|
const rawConfig = request.modelConfig ?? {};
|
|
106
116
|
return { ...request, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
|
|
107
117
|
}
|
|
118
|
+
function applyCandidateDeviceDefaults(request) {
|
|
119
|
+
const models = request.models;
|
|
120
|
+
if (models === undefined)
|
|
121
|
+
return request;
|
|
122
|
+
return {
|
|
123
|
+
...request,
|
|
124
|
+
models: models.map((load) => {
|
|
125
|
+
let canonicalType;
|
|
126
|
+
try {
|
|
127
|
+
canonicalType = normalizeModelType(load.modelType);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return load;
|
|
131
|
+
}
|
|
132
|
+
const rawConfig = load.modelConfig ?? {};
|
|
133
|
+
return { ...load, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
|
|
134
|
+
})
|
|
135
|
+
};
|
|
136
|
+
}
|
|
108
137
|
function getProfilingMeta(request) {
|
|
109
138
|
if (PROFILING_KEY in request) {
|
|
110
139
|
return request[PROFILING_KEY];
|
|
@@ -178,7 +207,10 @@ async function* streamWithProgress(request, handler) {
|
|
|
178
207
|
throw handlerError;
|
|
179
208
|
}
|
|
180
209
|
export async function send(request, _options) {
|
|
181
|
-
await ensureReady(
|
|
210
|
+
await ensureReady(request.type === 'startRpcServer' ||
|
|
211
|
+
request.type === 'stopRpcServer' ||
|
|
212
|
+
request.type === 'discoverRpcServers' ||
|
|
213
|
+
(request.type === 'cancel' && request.operation === 'request'));
|
|
182
214
|
assertLifecycleAllowed(request);
|
|
183
215
|
const processed = prepareRequest(request);
|
|
184
216
|
const entry = getHandlerEntry(processed.type);
|
package/dist/errors/index.d.ts
CHANGED
|
@@ -227,6 +227,7 @@ export declare class InferenceCancelledError extends QvacErrorBase {
|
|
|
227
227
|
readonly requestId: string;
|
|
228
228
|
readonly partial: InferenceCancelledPartial;
|
|
229
229
|
constructor(requestId: string, partial?: InferenceCancelledPartial, cause?: unknown);
|
|
230
|
+
toErrorResponseFields(): Record<string, unknown>;
|
|
230
231
|
}
|
|
231
232
|
export declare class AsyncDisposeUnavailableError extends QvacErrorBase {
|
|
232
233
|
constructor(cause?: unknown);
|
|
@@ -433,3 +434,9 @@ export declare class PathTraversalError extends QvacErrorBase {
|
|
|
433
434
|
export declare class ModelRegistryQueryFailedError extends QvacErrorBase {
|
|
434
435
|
constructor(details?: string, cause?: unknown);
|
|
435
436
|
}
|
|
437
|
+
export declare class RpcServerOperationError extends QvacErrorBase {
|
|
438
|
+
readonly operation: string;
|
|
439
|
+
readonly details: string;
|
|
440
|
+
constructor(operation: string, details: string, cause?: unknown);
|
|
441
|
+
toErrorResponseFields(): Record<string, unknown>;
|
|
442
|
+
}
|
package/dist/errors/index.js
CHANGED
|
@@ -354,6 +354,9 @@ export class InferenceCancelledError extends QvacErrorBase {
|
|
|
354
354
|
this.requestId = requestId;
|
|
355
355
|
this.partial = partial;
|
|
356
356
|
}
|
|
357
|
+
toErrorResponseFields() {
|
|
358
|
+
return { requestId: this.requestId, partial: this.partial };
|
|
359
|
+
}
|
|
357
360
|
}
|
|
358
361
|
export class AsyncDisposeUnavailableError extends QvacErrorBase {
|
|
359
362
|
constructor(cause) {
|
|
@@ -715,3 +718,15 @@ export class ModelRegistryQueryFailedError extends QvacErrorBase {
|
|
|
715
718
|
super(createErrorOptions(ERROR_CODES.QVAC_MODEL_REGISTRY_QUERY_FAILED, details ? [details] : undefined, cause));
|
|
716
719
|
}
|
|
717
720
|
}
|
|
721
|
+
export class RpcServerOperationError extends QvacErrorBase {
|
|
722
|
+
operation;
|
|
723
|
+
details;
|
|
724
|
+
constructor(operation, details, cause) {
|
|
725
|
+
super(createErrorOptions(ERROR_CODES.RPC_SERVER_OPERATION_FAILED, [operation, details], cause));
|
|
726
|
+
this.operation = operation;
|
|
727
|
+
this.details = details;
|
|
728
|
+
}
|
|
729
|
+
toErrorResponseFields() {
|
|
730
|
+
return { operation: this.operation, details: this.details };
|
|
731
|
+
}
|
|
732
|
+
}
|
|
@@ -1,10 +1,22 @@
|
|
|
1
|
-
import type { AssessModelFitRequest, AssessModelFitResponse } from '../schemas/assess-model-fit.js';
|
|
1
|
+
import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate, ModelFitEstimateTarget } from '../schemas/assess-model-fit.js';
|
|
2
2
|
/**
|
|
3
3
|
* Runs a pre-download fit assessment worker-side.
|
|
4
4
|
*
|
|
5
|
-
* This lives on the worker because that is where the
|
|
6
|
-
* already are: the resource collector for a fresh memory sample,
|
|
7
|
-
*
|
|
8
|
-
* loaded
|
|
5
|
+
* This lives on the worker because that is where the three things it needs
|
|
6
|
+
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
7
|
+
* own platform/arch pair, and the registry client. No weights are read and
|
|
8
|
+
* nothing is loaded — a single candidate additionally has the registry's
|
|
9
|
+
* weightless description fetched, tens of KB, so the engine's own fitter can
|
|
10
|
+
* answer instead of the coefficients modelling it.
|
|
9
11
|
*/
|
|
10
|
-
export declare function handleAssessModelFit(request: AssessModelFitRequest): AssessModelFitResponse
|
|
12
|
+
export declare function handleAssessModelFit(request: AssessModelFitRequest): Promise<AssessModelFitResponse>;
|
|
13
|
+
/**
|
|
14
|
+
* What the estimator is given for one candidate. The catalog facts come from
|
|
15
|
+
* the source's checksum, and the workload from the settings the load carries;
|
|
16
|
+
* a source outside the catalog resolves neither and assesses as `unknown`.
|
|
17
|
+
*
|
|
18
|
+
* A load whose sources are all config fields carries no checksum, so it has no
|
|
19
|
+
* catalog profile and the engine's own fitter is its only evidence. The model
|
|
20
|
+
* type stands in as the label.
|
|
21
|
+
*/
|
|
22
|
+
export declare function estimateTargetFor(candidate: ModelFitCandidate): ModelFitEstimateTarget;
|
|
@@ -1,26 +1,233 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { isCanonicalModelType, normalizeModelType, ModelType } from '../schemas/index.js';
|
|
2
|
+
import { inferModelTypeFromModelSrc } from '../schemas/model-src-utils.js';
|
|
3
|
+
import { ModelTypeRequiredError } from '../errors/index.js';
|
|
4
|
+
import { projectFitFromLoad } from '../resources/model-fit/fit-stub/project-fit-from-load.js';
|
|
2
5
|
import { getResourceCollector } from '../resources/instance.js';
|
|
6
|
+
import { getConfig } from '../runtime/state.js';
|
|
3
7
|
import { assessModelFitFromResources } from '../resources/model-fit/assess.js';
|
|
4
8
|
import { getPlatformCalibration } from '../resources/model-fit/calibration/index.js';
|
|
9
|
+
import { detectPlatform } from '../resources/model-fit/platform.js';
|
|
5
10
|
/**
|
|
6
11
|
* Runs a pre-download fit assessment worker-side.
|
|
7
12
|
*
|
|
8
|
-
* This lives on the worker because that is where the
|
|
9
|
-
* already are: the resource collector for a fresh memory sample,
|
|
10
|
-
*
|
|
11
|
-
* loaded
|
|
13
|
+
* This lives on the worker because that is where the three things it needs
|
|
14
|
+
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
15
|
+
* own platform/arch pair, and the registry client. No weights are read and
|
|
16
|
+
* nothing is loaded — a single candidate additionally has the registry's
|
|
17
|
+
* weightless description fetched, tens of KB, so the engine's own fitter can
|
|
18
|
+
* answer instead of the coefficients modelling it.
|
|
12
19
|
*/
|
|
13
|
-
export function handleAssessModelFit(request) {
|
|
20
|
+
export async function handleAssessModelFit(request) {
|
|
14
21
|
const platform = detectPlatform();
|
|
22
|
+
const native = await resolveNativeFit(request.models);
|
|
15
23
|
const result = assessModelFitFromResources({
|
|
16
|
-
models: request.models,
|
|
24
|
+
models: request.models.map(estimateTargetFor),
|
|
17
25
|
execution: request.execution,
|
|
18
26
|
resources: readResources(),
|
|
19
27
|
platform,
|
|
20
|
-
calibration: platform ? getPlatformCalibration(platform) : undefined
|
|
28
|
+
calibration: platform ? getPlatformCalibration(platform) : undefined,
|
|
29
|
+
nativeFit: native.fit,
|
|
30
|
+
nativeFitUnavailable: native.unavailable
|
|
21
31
|
});
|
|
22
32
|
return { type: 'assessModelFit', ...result };
|
|
23
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* The engine that would run this load, named outright or read off the source,
|
|
36
|
+
* as `loadModel` resolves it.
|
|
37
|
+
*
|
|
38
|
+
* @throws {ModelTypeRequiredError} When neither names one.
|
|
39
|
+
*/
|
|
40
|
+
function modelTypeOf(candidate) {
|
|
41
|
+
if (candidate.modelType !== undefined)
|
|
42
|
+
return normalizeModelType(candidate.modelType);
|
|
43
|
+
const inferred = inferModelTypeFromModelSrc(candidate.modelSrc);
|
|
44
|
+
if (inferred === undefined)
|
|
45
|
+
throw new ModelTypeRequiredError();
|
|
46
|
+
return normalizeModelType(inferred);
|
|
47
|
+
}
|
|
48
|
+
/** The audio engines, whose estimator sizes a load by its window rather than a context. */
|
|
49
|
+
const AUDIO_ENGINES = [
|
|
50
|
+
'whispercpp-transcription',
|
|
51
|
+
'parakeet-transcription',
|
|
52
|
+
'bci-whispercpp-transcription'
|
|
53
|
+
];
|
|
54
|
+
/**
|
|
55
|
+
* Engines that name a device outright, in llama's own `gpu` / `cpu` spelling.
|
|
56
|
+
* Both default to the GPU.
|
|
57
|
+
*/
|
|
58
|
+
const DEVICE_NAMED = [ModelType.llamacppCompletion, ModelType.llamacppEmbedding];
|
|
59
|
+
function contextUsesGpu(config) {
|
|
60
|
+
const contextParams = config['contextParams'];
|
|
61
|
+
if (contextParams === null || typeof contextParams !== 'object')
|
|
62
|
+
return undefined;
|
|
63
|
+
return contextParams['use_gpu'];
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Engines that carry a GPU switch, each read at the key its own config spells
|
|
67
|
+
* it under. These are the keys the fit builders read in `native-probe/engines`,
|
|
68
|
+
* so a projection and a device never disagree. Only bci defaults to on.
|
|
69
|
+
*/
|
|
70
|
+
const GPU_SWITCH = {
|
|
71
|
+
[ModelType.ttsGgml]: (config) => config['useGPU'] === true,
|
|
72
|
+
[ModelType.audiogenGgml]: (config) => config['useGPU'] === true,
|
|
73
|
+
[ModelType.parakeetTranscription]: (config) => config['useGPU'] === true,
|
|
74
|
+
[ModelType.whispercppTranscription]: (config) => contextUsesGpu(config) === true,
|
|
75
|
+
[ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Where a load resolved to run. `dispatch` applies the host's device defaults
|
|
79
|
+
* before any handler sees the config, so what arrives here is the resolved one.
|
|
80
|
+
*
|
|
81
|
+
* `nGpuLayers` wins over `useGPU` on tts, and the config schema rejects the two
|
|
82
|
+
* disagreeing, so reading it first is safe.
|
|
83
|
+
*/
|
|
84
|
+
function resolvedDevice(modelType, config) {
|
|
85
|
+
if (DEVICE_NAMED.includes(modelType)) {
|
|
86
|
+
const device = config['device'];
|
|
87
|
+
return typeof device === 'string' ? device.toLowerCase() : 'gpu';
|
|
88
|
+
}
|
|
89
|
+
const layers = config['nGpuLayers'];
|
|
90
|
+
if (modelType === ModelType.ttsGgml && typeof layers === 'number') {
|
|
91
|
+
return layers === 0 ? 'cpu' : 'gpu';
|
|
92
|
+
}
|
|
93
|
+
const usesGpu = GPU_SWITCH[modelType];
|
|
94
|
+
if (usesGpu === undefined)
|
|
95
|
+
return undefined;
|
|
96
|
+
return usesGpu(config) ? 'gpu' : 'cpu';
|
|
97
|
+
}
|
|
98
|
+
/** The window the speech engines hold whole; longer audio is chunked into it. */
|
|
99
|
+
const AUDIO_WINDOW_MS = 30_000;
|
|
100
|
+
/**
|
|
101
|
+
* The audio a single call holds in memory. `duration_ms` is the longest clip
|
|
102
|
+
* the caller will transcribe, not a per-call window, and the engine chunks
|
|
103
|
+
* anything longer, so it never sizes working memory above one window.
|
|
104
|
+
*/
|
|
105
|
+
function audioWindowMs(config) {
|
|
106
|
+
const declared = config['duration_ms'];
|
|
107
|
+
if (typeof declared !== 'number' || declared <= 0)
|
|
108
|
+
return AUDIO_WINDOW_MS;
|
|
109
|
+
return Math.min(declared, AUDIO_WINDOW_MS);
|
|
110
|
+
}
|
|
111
|
+
/** The catalog facts a source carries, where it carries them. */
|
|
112
|
+
function modelRefOf(value) {
|
|
113
|
+
if (value === null || typeof value !== 'object')
|
|
114
|
+
return undefined;
|
|
115
|
+
const src = value;
|
|
116
|
+
if (typeof src['sha256Checksum'] !== 'string')
|
|
117
|
+
return undefined;
|
|
118
|
+
const name = src['name'] ?? src['modelId'] ?? src['registryPath'];
|
|
119
|
+
return {
|
|
120
|
+
name: typeof name === 'string' ? name : '',
|
|
121
|
+
sha256Checksum: src['sha256Checksum'],
|
|
122
|
+
...(typeof src['registryPath'] === 'string' && { registryPath: src['registryPath'] }),
|
|
123
|
+
...(typeof src['registrySource'] === 'string' && { registrySource: src['registrySource'] })
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* The companion sources a load carries in its config — a VAD model, a pivot,
|
|
128
|
+
* an audiogen stage, a projector. Their bytes are part of what the set costs,
|
|
129
|
+
* so the estimate counts them alongside the primary.
|
|
130
|
+
*/
|
|
131
|
+
function companionRefs(config) {
|
|
132
|
+
const refs = [];
|
|
133
|
+
const seen = new Set();
|
|
134
|
+
const add = (ref) => {
|
|
135
|
+
if (seen.has(ref.sha256Checksum))
|
|
136
|
+
return;
|
|
137
|
+
seen.add(ref.sha256Checksum);
|
|
138
|
+
refs.push(ref);
|
|
139
|
+
};
|
|
140
|
+
const walk = (key, value) => {
|
|
141
|
+
const ref = modelRefOf(value);
|
|
142
|
+
if (ref !== undefined) {
|
|
143
|
+
add(ref);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (key.endsWith('ModelSrc')) {
|
|
147
|
+
add({ name: typeof value === 'string' ? value : key, sha256Checksum: '' });
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (value === null || typeof value !== 'object')
|
|
151
|
+
return;
|
|
152
|
+
for (const [inner, nested] of Object.entries(value)) {
|
|
153
|
+
walk(inner, nested);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
for (const [key, value] of Object.entries(config))
|
|
157
|
+
walk(key, value);
|
|
158
|
+
return refs;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* What the estimator is given for one candidate. The catalog facts come from
|
|
162
|
+
* the source's checksum, and the workload from the settings the load carries;
|
|
163
|
+
* a source outside the catalog resolves neither and assesses as `unknown`.
|
|
164
|
+
*
|
|
165
|
+
* A load whose sources are all config fields carries no checksum, so it has no
|
|
166
|
+
* catalog profile and the engine's own fitter is its only evidence. The model
|
|
167
|
+
* type stands in as the label.
|
|
168
|
+
*/
|
|
169
|
+
export function estimateTargetFor(candidate) {
|
|
170
|
+
const descriptor = typeof candidate.modelSrc === 'string' ? undefined : candidate.modelSrc;
|
|
171
|
+
const location = typeof candidate.modelSrc === 'string' ? candidate.modelSrc : undefined;
|
|
172
|
+
const config = candidate.modelConfig ?? {};
|
|
173
|
+
const modelType = modelTypeOf(candidate);
|
|
174
|
+
const contextTokens = config['ctx_size'];
|
|
175
|
+
const artifacts = companionRefs(config);
|
|
176
|
+
const device = resolvedDevice(modelType, config);
|
|
177
|
+
return {
|
|
178
|
+
model: {
|
|
179
|
+
name: descriptor?.name ?? descriptor?.modelId ?? location ?? modelType,
|
|
180
|
+
sha256Checksum: descriptor?.sha256Checksum ?? '',
|
|
181
|
+
...(descriptor?.registryPath !== undefined && { registryPath: descriptor.registryPath }),
|
|
182
|
+
...(descriptor?.registrySource !== undefined && {
|
|
183
|
+
registrySource: descriptor.registrySource
|
|
184
|
+
})
|
|
185
|
+
},
|
|
186
|
+
...(artifacts.length > 0 && { artifacts }),
|
|
187
|
+
...(device !== undefined && { device }),
|
|
188
|
+
workload: AUDIO_ENGINES.includes(modelType)
|
|
189
|
+
? {
|
|
190
|
+
kind: 'audio',
|
|
191
|
+
windowMs: audioWindowMs(config),
|
|
192
|
+
streaming: config['streaming'] === true
|
|
193
|
+
}
|
|
194
|
+
: {
|
|
195
|
+
kind: 'llm',
|
|
196
|
+
...(typeof contextTokens === 'number' && contextTokens > 0 && { contextTokens })
|
|
197
|
+
}
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* The engine fitter's verdict for a single candidate, resolved through that
|
|
202
|
+
* load's own plugin. Only for a one-candidate request: the probe measures one
|
|
203
|
+
* model against the whole machine, which cannot be aggregated across a set.
|
|
204
|
+
*/
|
|
205
|
+
async function resolveNativeFit(candidates) {
|
|
206
|
+
if (candidates.length !== 1)
|
|
207
|
+
return {};
|
|
208
|
+
const candidate = candidates[0];
|
|
209
|
+
if (!candidate)
|
|
210
|
+
return {};
|
|
211
|
+
const modelType = modelTypeOf(candidate);
|
|
212
|
+
if (!isCanonicalModelType(modelType)) {
|
|
213
|
+
return { unavailable: `no plugin handles model type ${modelType}` };
|
|
214
|
+
}
|
|
215
|
+
const budgetMs = getConfig().fitStubBudgetMs;
|
|
216
|
+
const outcome = await projectFitFromLoad({
|
|
217
|
+
modelType,
|
|
218
|
+
...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
|
|
219
|
+
...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig })
|
|
220
|
+
}, estimateTargetFor(candidate).model.name, budgetMs === undefined ? {} : { stub: { budgetMs } });
|
|
221
|
+
if (outcome.status === 'projected')
|
|
222
|
+
return { fit: outcome.fit };
|
|
223
|
+
if (outcome.status === 'unsupported-load')
|
|
224
|
+
return { unavailable: outcome.detail };
|
|
225
|
+
return {
|
|
226
|
+
unavailable: outcome.message === undefined
|
|
227
|
+
? `no registry description (${outcome.reason})`
|
|
228
|
+
: `no registry description (${outcome.reason}): ${outcome.message}`
|
|
229
|
+
};
|
|
230
|
+
}
|
|
24
231
|
function readResources() {
|
|
25
232
|
const collector = getResourceCollector();
|
|
26
233
|
if (!collector) {
|
|
@@ -46,30 +253,3 @@ function readResources() {
|
|
|
46
253
|
}
|
|
47
254
|
return { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
48
255
|
}
|
|
49
|
-
/**
|
|
50
|
-
* Maps the runtime's platform and architecture onto a calibration target.
|
|
51
|
-
*
|
|
52
|
-
* @returns `undefined` for any pair this feature has no calibration target for,
|
|
53
|
-
* which assesses as `unknown` rather than borrowing another platform's
|
|
54
|
-
* coefficients.
|
|
55
|
-
*/
|
|
56
|
-
function detectPlatform() {
|
|
57
|
-
const key = `${os.platform()}-${os.arch()}`;
|
|
58
|
-
switch (key) {
|
|
59
|
-
case 'darwin-arm64':
|
|
60
|
-
case 'darwin-x64':
|
|
61
|
-
case 'linux-arm64':
|
|
62
|
-
case 'linux-x64':
|
|
63
|
-
case 'win32-x64':
|
|
64
|
-
// Listed for completeness rather than reach: no engine addon is built for
|
|
65
|
-
// win32-arm64 (`@qvac/llm-llamacpp/prebuilds` has no such target, and no
|
|
66
|
-
// windows-arm runner exists to build one), so nothing on that platform can
|
|
67
|
-
// load a model today. Calibrating it needs that build first, not a fixture.
|
|
68
|
-
case 'win32-arm64':
|
|
69
|
-
case 'android-arm64':
|
|
70
|
-
case 'ios-arm64':
|
|
71
|
-
return key;
|
|
72
|
-
default:
|
|
73
|
-
return undefined;
|
|
74
|
-
}
|
|
75
|
-
}
|
|
@@ -2,6 +2,7 @@ import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
|
|
|
2
2
|
import { aggregateEvents } from '../utils/aggregate-events.js';
|
|
3
3
|
import { CompletionFailedError } from '../errors/index.js';
|
|
4
4
|
import { dispatchPluginStream } from './plugin-dispatch.js';
|
|
5
|
+
import { executeToolSearch, TOOL_SEARCH_NAME } from '../utils/tools/defer.js';
|
|
5
6
|
const DEFAULT_MAX_TOOL_TURNS = 8;
|
|
6
7
|
// Generous ceiling on one client-side tool execution: long enough for a slow
|
|
7
8
|
// MCP round trip, short enough that a client that stopped reading can't pin a
|
|
@@ -154,7 +155,23 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
154
155
|
// The assistant's tool-call turn goes back verbatim so the model sees
|
|
155
156
|
// its own call syntax; each result follows as a `tool` message.
|
|
156
157
|
history = [...history, { role: 'assistant', content: rawFullText ?? contentText }];
|
|
158
|
+
// Without a deferred tool, a `tool_search` call is the caller's own tool.
|
|
159
|
+
const defers = request.tools?.some((tool) => tool.deferLoading === true) ?? false;
|
|
157
160
|
for (const call of toolCalls) {
|
|
161
|
+
// `tool_search` is the SDK's own tool: it reads the registered inventory
|
|
162
|
+
// and appends the matched definitions, so it never leaves the worker and
|
|
163
|
+
// the client is never asked to run it. The `toolCall` event still went
|
|
164
|
+
// downstream with the turn's events, so the client can see it happened.
|
|
165
|
+
if (defers && call.name === TOOL_SEARCH_NAME) {
|
|
166
|
+
history = [
|
|
167
|
+
...history,
|
|
168
|
+
{
|
|
169
|
+
role: 'tool',
|
|
170
|
+
content: executeToolSearch(request.tools ?? [], call.arguments, history)
|
|
171
|
+
}
|
|
172
|
+
];
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
158
175
|
yield {
|
|
159
176
|
type: 'completionOrchestrate',
|
|
160
177
|
turn,
|
|
@@ -15,6 +15,8 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
15
15
|
logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
|
|
16
16
|
}
|
|
17
17
|
const handlers = plugin ? Object.keys(plugin.handlers) : [];
|
|
18
|
+
const fitProbe = entry.local.fitProbe;
|
|
19
|
+
const probeRan = fitProbe !== undefined && fitProbe.reason !== 'disabled';
|
|
18
20
|
const info = {
|
|
19
21
|
modelId: entry.id,
|
|
20
22
|
modelType: entry.local.modelType,
|
|
@@ -27,7 +29,8 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
27
29
|
// Same detection the completion normalizer uses, so callers see the dialect it parses.
|
|
28
30
|
...(handlers.includes('completionStream') && {
|
|
29
31
|
toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
|
|
30
|
-
})
|
|
32
|
+
}),
|
|
33
|
+
...(probeRan && { fitProbe })
|
|
31
34
|
};
|
|
32
35
|
return { type: 'getLoadedModelInfo', info };
|
|
33
36
|
}
|