@qvac/inference 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +90 -24
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
|
@@ -6,9 +6,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
6
6
|
init_image?: string;
|
|
7
7
|
control_frames?: string[];
|
|
8
8
|
modelId: string;
|
|
9
|
+
prompt: string;
|
|
9
10
|
seed?: number | undefined;
|
|
10
11
|
vae_tiling?: boolean | undefined;
|
|
11
|
-
prompt: string;
|
|
12
12
|
negative_prompt?: string | undefined;
|
|
13
13
|
width?: number | undefined;
|
|
14
14
|
height?: number | undefined;
|
|
@@ -47,9 +47,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
47
47
|
init_image?: string;
|
|
48
48
|
control_frames?: string[];
|
|
49
49
|
modelId: string;
|
|
50
|
+
prompt: string;
|
|
50
51
|
seed?: number | undefined;
|
|
51
52
|
vae_tiling?: boolean | undefined;
|
|
52
|
-
prompt: string;
|
|
53
53
|
negative_prompt?: string | undefined;
|
|
54
54
|
width?: number | undefined;
|
|
55
55
|
height?: number | undefined;
|
|
@@ -88,9 +88,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
88
88
|
init_image?: string;
|
|
89
89
|
control_frames?: string[];
|
|
90
90
|
modelId: string;
|
|
91
|
+
prompt: string;
|
|
91
92
|
seed?: number | undefined;
|
|
92
93
|
vae_tiling?: boolean | undefined;
|
|
93
|
-
prompt: string;
|
|
94
94
|
negative_prompt?: string | undefined;
|
|
95
95
|
width?: number | undefined;
|
|
96
96
|
height?: number | undefined;
|
|
@@ -129,9 +129,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
129
129
|
init_image?: string;
|
|
130
130
|
control_frames?: string[];
|
|
131
131
|
modelId: string;
|
|
132
|
+
prompt: string;
|
|
132
133
|
seed?: number | undefined;
|
|
133
134
|
vae_tiling?: boolean | undefined;
|
|
134
|
-
prompt: string;
|
|
135
135
|
negative_prompt?: string | undefined;
|
|
136
136
|
width?: number | undefined;
|
|
137
137
|
height?: number | undefined;
|
|
@@ -170,9 +170,9 @@ export declare function createVideoStreamRequest(params: VideoClientParams, requ
|
|
|
170
170
|
init_image?: string;
|
|
171
171
|
control_frames?: string[];
|
|
172
172
|
modelId: string;
|
|
173
|
+
prompt: string;
|
|
173
174
|
seed?: number | undefined;
|
|
174
175
|
vae_tiling?: boolean | undefined;
|
|
175
|
-
prompt: string;
|
|
176
176
|
negative_prompt?: string | undefined;
|
|
177
177
|
width?: number | undefined;
|
|
178
178
|
height?: number | undefined;
|
package/dist/dispatch.js
CHANGED
|
@@ -9,6 +9,7 @@ import { assertLifecycleAllowed } from './runtime/runtime-lifecycle.js';
|
|
|
9
9
|
import { resolveModelConfig, setConfig, setRuntimeContext, isConfigSet } from './runtime/state.js';
|
|
10
10
|
import { initialize, close as closeEngine } from './runtime/lifecycle.js';
|
|
11
11
|
import { getAllPlugins } from './plugins/index.js';
|
|
12
|
+
import { hasRpcServerProvider } from './rpc/provider.js';
|
|
12
13
|
import { resolveConfig } from './config/resolve-config.js';
|
|
13
14
|
import { setGlobalLogLevel, setGlobalConsoleOutput, getAppLogger } from './logging/index.js';
|
|
14
15
|
import { profileReplyHandler, profileStreamHandler } from './profiling/index.js';
|
|
@@ -24,7 +25,7 @@ let readyPromise = null;
|
|
|
24
25
|
// does not mark readiness.
|
|
25
26
|
let generation = 0;
|
|
26
27
|
function ensurePluginsRegistered() {
|
|
27
|
-
if (getAllPlugins().length === 0) {
|
|
28
|
+
if (getAllPlugins().length === 0 && !hasRpcServerProvider()) {
|
|
28
29
|
throw new PluginsNotRegisteredError();
|
|
29
30
|
}
|
|
30
31
|
}
|
|
@@ -56,14 +57,15 @@ async function initializeConfig() {
|
|
|
56
57
|
}
|
|
57
58
|
async function performReady(gen) {
|
|
58
59
|
initialize();
|
|
59
|
-
ensurePluginsRegistered();
|
|
60
60
|
await initializeConfig();
|
|
61
61
|
// Mark ready only while this generation is current — a `close()` during the
|
|
62
62
|
// await bumps it past this now-superseded init.
|
|
63
63
|
if (gen === generation)
|
|
64
64
|
ready = true;
|
|
65
65
|
}
|
|
66
|
-
async function ensureReady() {
|
|
66
|
+
async function ensureReady(allowWithoutPlugins = false) {
|
|
67
|
+
if (!allowWithoutPlugins)
|
|
68
|
+
ensurePluginsRegistered();
|
|
67
69
|
if (ready)
|
|
68
70
|
return;
|
|
69
71
|
// A single shared promise serializes concurrent first calls: `ready` only
|
|
@@ -88,11 +90,19 @@ function getHandlerEntry(type) {
|
|
|
88
90
|
return entry;
|
|
89
91
|
}
|
|
90
92
|
/**
|
|
91
|
-
*
|
|
92
|
-
* handler runs, matching the priority user config >
|
|
93
|
-
* defaults
|
|
93
|
+
* Fills a request that describes a load with device and schema config
|
|
94
|
+
* defaults before the handler runs, matching the priority user config >
|
|
95
|
+
* device defaults > schema defaults.
|
|
96
|
+
*
|
|
97
|
+
* `loadModel` and every `assessModelFit` candidate describe the same load, so
|
|
98
|
+
* both are filled the same way: the fitter is asked about the settings a real
|
|
99
|
+
* load resolves, and refuses one carrying no `device` at all. Every other
|
|
100
|
+
* request type passes through untouched.
|
|
94
101
|
*/
|
|
95
102
|
function applyDeviceDefaults(request) {
|
|
103
|
+
if (request.type === 'assessModelFit' && 'models' in request) {
|
|
104
|
+
return applyCandidateDeviceDefaults(request);
|
|
105
|
+
}
|
|
96
106
|
if (request.type !== 'loadModel' || !('modelSrc' in request))
|
|
97
107
|
return request;
|
|
98
108
|
let canonicalType;
|
|
@@ -105,6 +115,25 @@ function applyDeviceDefaults(request) {
|
|
|
105
115
|
const rawConfig = request.modelConfig ?? {};
|
|
106
116
|
return { ...request, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
|
|
107
117
|
}
|
|
118
|
+
function applyCandidateDeviceDefaults(request) {
|
|
119
|
+
const models = request.models;
|
|
120
|
+
if (models === undefined)
|
|
121
|
+
return request;
|
|
122
|
+
return {
|
|
123
|
+
...request,
|
|
124
|
+
models: models.map((load) => {
|
|
125
|
+
let canonicalType;
|
|
126
|
+
try {
|
|
127
|
+
canonicalType = normalizeModelType(load.modelType);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return load;
|
|
131
|
+
}
|
|
132
|
+
const rawConfig = load.modelConfig ?? {};
|
|
133
|
+
return { ...load, modelConfig: resolveModelConfig(canonicalType, rawConfig) };
|
|
134
|
+
})
|
|
135
|
+
};
|
|
136
|
+
}
|
|
108
137
|
function getProfilingMeta(request) {
|
|
109
138
|
if (PROFILING_KEY in request) {
|
|
110
139
|
return request[PROFILING_KEY];
|
|
@@ -178,7 +207,10 @@ async function* streamWithProgress(request, handler) {
|
|
|
178
207
|
throw handlerError;
|
|
179
208
|
}
|
|
180
209
|
export async function send(request, _options) {
|
|
181
|
-
await ensureReady(
|
|
210
|
+
await ensureReady(request.type === 'startRpcServer' ||
|
|
211
|
+
request.type === 'stopRpcServer' ||
|
|
212
|
+
request.type === 'discoverRpcServers' ||
|
|
213
|
+
(request.type === 'cancel' && request.operation === 'request'));
|
|
182
214
|
assertLifecycleAllowed(request);
|
|
183
215
|
const processed = prepareRequest(request);
|
|
184
216
|
const entry = getHandlerEntry(processed.type);
|
package/dist/errors/index.d.ts
CHANGED
|
@@ -227,6 +227,7 @@ export declare class InferenceCancelledError extends QvacErrorBase {
|
|
|
227
227
|
readonly requestId: string;
|
|
228
228
|
readonly partial: InferenceCancelledPartial;
|
|
229
229
|
constructor(requestId: string, partial?: InferenceCancelledPartial, cause?: unknown);
|
|
230
|
+
toErrorResponseFields(): Record<string, unknown>;
|
|
230
231
|
}
|
|
231
232
|
export declare class AsyncDisposeUnavailableError extends QvacErrorBase {
|
|
232
233
|
constructor(cause?: unknown);
|
|
@@ -433,3 +434,9 @@ export declare class PathTraversalError extends QvacErrorBase {
|
|
|
433
434
|
export declare class ModelRegistryQueryFailedError extends QvacErrorBase {
|
|
434
435
|
constructor(details?: string, cause?: unknown);
|
|
435
436
|
}
|
|
437
|
+
export declare class RpcServerOperationError extends QvacErrorBase {
|
|
438
|
+
readonly operation: string;
|
|
439
|
+
readonly details: string;
|
|
440
|
+
constructor(operation: string, details: string, cause?: unknown);
|
|
441
|
+
toErrorResponseFields(): Record<string, unknown>;
|
|
442
|
+
}
|
package/dist/errors/index.js
CHANGED
|
@@ -354,6 +354,9 @@ export class InferenceCancelledError extends QvacErrorBase {
|
|
|
354
354
|
this.requestId = requestId;
|
|
355
355
|
this.partial = partial;
|
|
356
356
|
}
|
|
357
|
+
toErrorResponseFields() {
|
|
358
|
+
return { requestId: this.requestId, partial: this.partial };
|
|
359
|
+
}
|
|
357
360
|
}
|
|
358
361
|
export class AsyncDisposeUnavailableError extends QvacErrorBase {
|
|
359
362
|
constructor(cause) {
|
|
@@ -715,3 +718,15 @@ export class ModelRegistryQueryFailedError extends QvacErrorBase {
|
|
|
715
718
|
super(createErrorOptions(ERROR_CODES.QVAC_MODEL_REGISTRY_QUERY_FAILED, details ? [details] : undefined, cause));
|
|
716
719
|
}
|
|
717
720
|
}
|
|
721
|
+
export class RpcServerOperationError extends QvacErrorBase {
|
|
722
|
+
operation;
|
|
723
|
+
details;
|
|
724
|
+
constructor(operation, details, cause) {
|
|
725
|
+
super(createErrorOptions(ERROR_CODES.RPC_SERVER_OPERATION_FAILED, [operation, details], cause));
|
|
726
|
+
this.operation = operation;
|
|
727
|
+
this.details = details;
|
|
728
|
+
}
|
|
729
|
+
toErrorResponseFields() {
|
|
730
|
+
return { operation: this.operation, details: this.details };
|
|
731
|
+
}
|
|
732
|
+
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate } from '../schemas/assess-model-fit.js';
|
|
2
|
-
import { type CanonicalModelType } from '../schemas/index.js';
|
|
3
|
-
import { type ProfileResolver } from '../resources/model-fit/assess.js';
|
|
1
|
+
import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate, ModelFitEstimateTarget } from '../schemas/assess-model-fit.js';
|
|
4
2
|
/**
|
|
5
3
|
* Runs a pre-download fit assessment worker-side.
|
|
6
4
|
*
|
|
@@ -12,23 +10,13 @@ import { type ProfileResolver } from '../resources/model-fit/assess.js';
|
|
|
12
10
|
* answer instead of the coefficients modelling it.
|
|
13
11
|
*/
|
|
14
12
|
export declare function handleAssessModelFit(request: AssessModelFitRequest): Promise<AssessModelFitResponse>;
|
|
15
|
-
export interface FitLoad {
|
|
16
|
-
modelType: CanonicalModelType;
|
|
17
|
-
modelConfig: Record<string, unknown>;
|
|
18
|
-
}
|
|
19
13
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
14
|
+
* What the estimator is given for one candidate. The catalog facts come from
|
|
15
|
+
* the source's checksum, and the workload from the settings the load carries;
|
|
16
|
+
* a source outside the catalog resolves neither and assesses as `unknown`.
|
|
22
17
|
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* config carries the same device defaults a real load resolves — the fitter
|
|
27
|
-
* answers `unsupported-config` for a load with no `device`. An embedding load
|
|
28
|
-
* has no context setting; the fitter reads the window the model declares.
|
|
29
|
-
*
|
|
30
|
-
* A candidate with companion `artifacts` gets no fitter: the estimator counts
|
|
31
|
-
* their bytes, the fitter reads one file, and a verdict for the model alone
|
|
32
|
-
* would stand in for the set.
|
|
18
|
+
* A load whose sources are all config fields carries no checksum, so it has no
|
|
19
|
+
* catalog profile and the engine's own fitter is its only evidence. The model
|
|
20
|
+
* type stands in as the label.
|
|
33
21
|
*/
|
|
34
|
-
export declare function
|
|
22
|
+
export declare function estimateTargetFor(candidate: ModelFitCandidate): ModelFitEstimateTarget;
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { getModelResourceProfile } from '../models/registry/resource-profiles.js';
|
|
1
|
+
import { isCanonicalModelType, normalizeModelType, ModelType } from '../schemas/index.js';
|
|
2
|
+
import { inferModelTypeFromModelSrc } from '../schemas/model-src-utils.js';
|
|
3
|
+
import { ModelTypeRequiredError } from '../errors/index.js';
|
|
4
|
+
import { projectFitFromLoad } from '../resources/model-fit/fit-stub/project-fit-from-load.js';
|
|
6
5
|
import { getResourceCollector } from '../resources/instance.js';
|
|
6
|
+
import { getConfig } from '../runtime/state.js';
|
|
7
7
|
import { assessModelFitFromResources } from '../resources/model-fit/assess.js';
|
|
8
8
|
import { getPlatformCalibration } from '../resources/model-fit/calibration/index.js';
|
|
9
|
+
import { detectPlatform } from '../resources/model-fit/platform.js';
|
|
9
10
|
/**
|
|
10
11
|
* Runs a pre-download fit assessment worker-side.
|
|
11
12
|
*
|
|
@@ -18,70 +19,214 @@ import { getPlatformCalibration } from '../resources/model-fit/calibration/index
|
|
|
18
19
|
*/
|
|
19
20
|
export async function handleAssessModelFit(request) {
|
|
20
21
|
const platform = detectPlatform();
|
|
22
|
+
const native = await resolveNativeFit(request.models);
|
|
21
23
|
const result = assessModelFitFromResources({
|
|
22
|
-
models: request.models,
|
|
24
|
+
models: request.models.map(estimateTargetFor),
|
|
23
25
|
execution: request.execution,
|
|
24
26
|
resources: readResources(),
|
|
25
27
|
platform,
|
|
26
28
|
calibration: platform ? getPlatformCalibration(platform) : undefined,
|
|
27
|
-
nativeFit:
|
|
29
|
+
nativeFit: native.fit,
|
|
30
|
+
nativeFitUnavailable: native.unavailable
|
|
28
31
|
});
|
|
29
32
|
return { type: 'assessModelFit', ...result };
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
|
-
* The engine
|
|
33
|
-
*
|
|
35
|
+
* The engine that would run this load, named outright or read off the source,
|
|
36
|
+
* as `loadModel` resolves it.
|
|
34
37
|
*
|
|
35
|
-
*
|
|
36
|
-
* whole machine, which cannot be aggregated across a set. Everything else — no
|
|
37
|
-
* stub published, an offline caller, an engine with no fit path — resolves to
|
|
38
|
-
* `undefined` and the modelled assessment stands.
|
|
38
|
+
* @throws {ModelTypeRequiredError} When neither names one.
|
|
39
39
|
*/
|
|
40
|
-
|
|
41
|
-
if (
|
|
42
|
-
return
|
|
43
|
-
const
|
|
44
|
-
if (
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
function modelTypeOf(candidate) {
|
|
41
|
+
if (candidate.modelType !== undefined)
|
|
42
|
+
return normalizeModelType(candidate.modelType);
|
|
43
|
+
const inferred = inferModelTypeFromModelSrc(candidate.modelSrc);
|
|
44
|
+
if (inferred === undefined)
|
|
45
|
+
throw new ModelTypeRequiredError();
|
|
46
|
+
return normalizeModelType(inferred);
|
|
47
|
+
}
|
|
48
|
+
/** The audio engines, whose estimator sizes a load by its window rather than a context. */
|
|
49
|
+
const AUDIO_ENGINES = [
|
|
50
|
+
'whispercpp-transcription',
|
|
51
|
+
'parakeet-transcription',
|
|
52
|
+
'bci-whispercpp-transcription'
|
|
53
|
+
];
|
|
54
|
+
/**
|
|
55
|
+
* Engines that name a device outright, in llama's own `gpu` / `cpu` spelling.
|
|
56
|
+
* Both default to the GPU.
|
|
57
|
+
*/
|
|
58
|
+
const DEVICE_NAMED = [ModelType.llamacppCompletion, ModelType.llamacppEmbedding];
|
|
59
|
+
function contextUsesGpu(config) {
|
|
60
|
+
const contextParams = config['contextParams'];
|
|
61
|
+
if (contextParams === null || typeof contextParams !== 'object')
|
|
48
62
|
return undefined;
|
|
49
|
-
|
|
50
|
-
return outcome.status === 'projected' ? outcome.fit : undefined;
|
|
63
|
+
return contextParams['use_gpu'];
|
|
51
64
|
}
|
|
52
65
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
66
|
+
* Engines that carry a GPU switch, each read at the key its own config spells
|
|
67
|
+
* it under. These are the keys the fit builders read in `native-probe/engines`,
|
|
68
|
+
* so a projection and a device never disagree. Only bci defaults to on.
|
|
69
|
+
*/
|
|
70
|
+
const GPU_SWITCH = {
|
|
71
|
+
[ModelType.ttsGgml]: (config) => config['useGPU'] === true,
|
|
72
|
+
[ModelType.audiogenGgml]: (config) => config['useGPU'] === true,
|
|
73
|
+
[ModelType.parakeetTranscription]: (config) => config['useGPU'] === true,
|
|
74
|
+
[ModelType.whispercppTranscription]: (config) => contextUsesGpu(config) === true,
|
|
75
|
+
[ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Where a load resolved to run. `dispatch` applies the host's device defaults
|
|
79
|
+
* before any handler sees the config, so what arrives here is the resolved one.
|
|
62
80
|
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
* would stand in for the set.
|
|
81
|
+
* `nGpuLayers` wins over `useGPU` on tts, and the config schema rejects the two
|
|
82
|
+
* disagreeing, so reading it first is safe.
|
|
66
83
|
*/
|
|
67
|
-
|
|
68
|
-
if (
|
|
84
|
+
function resolvedDevice(modelType, config) {
|
|
85
|
+
if (DEVICE_NAMED.includes(modelType)) {
|
|
86
|
+
const device = config['device'];
|
|
87
|
+
return typeof device === 'string' ? device.toLowerCase() : 'gpu';
|
|
88
|
+
}
|
|
89
|
+
const layers = config['nGpuLayers'];
|
|
90
|
+
if (modelType === ModelType.ttsGgml && typeof layers === 'number') {
|
|
91
|
+
return layers === 0 ? 'cpu' : 'gpu';
|
|
92
|
+
}
|
|
93
|
+
const usesGpu = GPU_SWITCH[modelType];
|
|
94
|
+
if (usesGpu === undefined)
|
|
69
95
|
return undefined;
|
|
70
|
-
|
|
96
|
+
return usesGpu(config) ? 'gpu' : 'cpu';
|
|
97
|
+
}
|
|
98
|
+
/** The window the speech engines hold whole; longer audio is chunked into it. */
|
|
99
|
+
const AUDIO_WINDOW_MS = 30_000;
|
|
100
|
+
/**
|
|
101
|
+
* The audio a single call holds in memory. `duration_ms` is the longest clip
|
|
102
|
+
* the caller will transcribe, not a per-call window, and the engine chunks
|
|
103
|
+
* anything longer, so it never sizes working memory above one window.
|
|
104
|
+
*/
|
|
105
|
+
function audioWindowMs(config) {
|
|
106
|
+
const declared = config['duration_ms'];
|
|
107
|
+
if (typeof declared !== 'number' || declared <= 0)
|
|
108
|
+
return AUDIO_WINDOW_MS;
|
|
109
|
+
return Math.min(declared, AUDIO_WINDOW_MS);
|
|
110
|
+
}
|
|
111
|
+
/** The catalog facts a source carries, where it carries them. */
|
|
112
|
+
function modelRefOf(value) {
|
|
113
|
+
if (value === null || typeof value !== 'object')
|
|
71
114
|
return undefined;
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
115
|
+
const src = value;
|
|
116
|
+
if (typeof src['sha256Checksum'] !== 'string')
|
|
117
|
+
return undefined;
|
|
118
|
+
const name = src['name'] ?? src['modelId'] ?? src['registryPath'];
|
|
119
|
+
return {
|
|
120
|
+
name: typeof name === 'string' ? name : '',
|
|
121
|
+
sha256Checksum: src['sha256Checksum'],
|
|
122
|
+
...(typeof src['registryPath'] === 'string' && { registryPath: src['registryPath'] }),
|
|
123
|
+
...(typeof src['registrySource'] === 'string' && { registrySource: src['registrySource'] })
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* The companion sources a load carries in its config — a VAD model, a pivot,
|
|
128
|
+
* an audiogen stage, a projector. Their bytes are part of what the set costs,
|
|
129
|
+
* so the estimate counts them alongside the primary.
|
|
130
|
+
*/
|
|
131
|
+
function companionRefs(config) {
|
|
132
|
+
const refs = [];
|
|
133
|
+
const seen = new Set();
|
|
134
|
+
const add = (ref) => {
|
|
135
|
+
if (seen.has(ref.sha256Checksum))
|
|
136
|
+
return;
|
|
137
|
+
seen.add(ref.sha256Checksum);
|
|
138
|
+
refs.push(ref);
|
|
139
|
+
};
|
|
140
|
+
const walk = (key, value) => {
|
|
141
|
+
const ref = modelRefOf(value);
|
|
142
|
+
if (ref !== undefined) {
|
|
143
|
+
add(ref);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (key.endsWith('ModelSrc')) {
|
|
147
|
+
add({ name: typeof value === 'string' ? value : key, sha256Checksum: '' });
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (value === null || typeof value !== 'object')
|
|
151
|
+
return;
|
|
152
|
+
for (const [inner, nested] of Object.entries(value)) {
|
|
153
|
+
walk(inner, nested);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
for (const [key, value] of Object.entries(config))
|
|
157
|
+
walk(key, value);
|
|
158
|
+
return refs;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* What the estimator is given for one candidate. The catalog facts come from
|
|
162
|
+
* the source's checksum, and the workload from the settings the load carries;
|
|
163
|
+
* a source outside the catalog resolves neither and assesses as `unknown`.
|
|
164
|
+
*
|
|
165
|
+
* A load whose sources are all config fields carries no checksum, so it has no
|
|
166
|
+
* catalog profile and the engine's own fitter is its only evidence. The model
|
|
167
|
+
* type stands in as the label.
|
|
168
|
+
*/
|
|
169
|
+
export function estimateTargetFor(candidate) {
|
|
170
|
+
const descriptor = typeof candidate.modelSrc === 'string' ? undefined : candidate.modelSrc;
|
|
171
|
+
const location = typeof candidate.modelSrc === 'string' ? candidate.modelSrc : undefined;
|
|
172
|
+
const config = candidate.modelConfig ?? {};
|
|
173
|
+
const modelType = modelTypeOf(candidate);
|
|
174
|
+
const contextTokens = config['ctx_size'];
|
|
175
|
+
const artifacts = companionRefs(config);
|
|
176
|
+
const device = resolvedDevice(modelType, config);
|
|
177
|
+
return {
|
|
178
|
+
model: {
|
|
179
|
+
name: descriptor?.name ?? descriptor?.modelId ?? location ?? modelType,
|
|
180
|
+
sha256Checksum: descriptor?.sha256Checksum ?? '',
|
|
181
|
+
...(descriptor?.registryPath !== undefined && { registryPath: descriptor.registryPath }),
|
|
182
|
+
...(descriptor?.registrySource !== undefined && {
|
|
183
|
+
registrySource: descriptor.registrySource
|
|
184
|
+
})
|
|
185
|
+
},
|
|
186
|
+
...(artifacts.length > 0 && { artifacts }),
|
|
187
|
+
...(device !== undefined && { device }),
|
|
188
|
+
workload: AUDIO_ENGINES.includes(modelType)
|
|
189
|
+
? {
|
|
190
|
+
kind: 'audio',
|
|
191
|
+
windowMs: audioWindowMs(config),
|
|
192
|
+
streaming: config['streaming'] === true
|
|
193
|
+
}
|
|
194
|
+
: {
|
|
195
|
+
kind: 'llm',
|
|
196
|
+
...(typeof contextTokens === 'number' && contextTokens > 0 && { contextTokens })
|
|
197
|
+
}
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* The engine fitter's verdict for a single candidate, resolved through that
|
|
202
|
+
* load's own plugin. Only for a one-candidate request: the probe measures one
|
|
203
|
+
* model against the whole machine, which cannot be aggregated across a set.
|
|
204
|
+
*/
|
|
205
|
+
async function resolveNativeFit(candidates) {
|
|
206
|
+
if (candidates.length !== 1)
|
|
207
|
+
return {};
|
|
208
|
+
const candidate = candidates[0];
|
|
209
|
+
if (!candidate)
|
|
210
|
+
return {};
|
|
211
|
+
const modelType = modelTypeOf(candidate);
|
|
212
|
+
if (!isCanonicalModelType(modelType)) {
|
|
213
|
+
return { unavailable: `no plugin handles model type ${modelType}` };
|
|
83
214
|
}
|
|
84
|
-
|
|
215
|
+
const budgetMs = getConfig().fitStubBudgetMs;
|
|
216
|
+
const outcome = await projectFitFromLoad({
|
|
217
|
+
modelType,
|
|
218
|
+
...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
|
|
219
|
+
...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig })
|
|
220
|
+
}, estimateTargetFor(candidate).model.name, budgetMs === undefined ? {} : { stub: { budgetMs } });
|
|
221
|
+
if (outcome.status === 'projected')
|
|
222
|
+
return { fit: outcome.fit };
|
|
223
|
+
if (outcome.status === 'unsupported-load')
|
|
224
|
+
return { unavailable: outcome.detail };
|
|
225
|
+
return {
|
|
226
|
+
unavailable: outcome.message === undefined
|
|
227
|
+
? `no registry description (${outcome.reason})`
|
|
228
|
+
: `no registry description (${outcome.reason}): ${outcome.message}`
|
|
229
|
+
};
|
|
85
230
|
}
|
|
86
231
|
function readResources() {
|
|
87
232
|
const collector = getResourceCollector();
|
|
@@ -108,30 +253,3 @@ function readResources() {
|
|
|
108
253
|
}
|
|
109
254
|
return { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
110
255
|
}
|
|
111
|
-
/**
|
|
112
|
-
* Maps the runtime's platform and architecture onto a calibration target.
|
|
113
|
-
*
|
|
114
|
-
* @returns `undefined` for any pair this feature has no calibration target for,
|
|
115
|
-
* which assesses as `unknown` rather than borrowing another platform's
|
|
116
|
-
* coefficients.
|
|
117
|
-
*/
|
|
118
|
-
function detectPlatform() {
|
|
119
|
-
const key = `${os.platform()}-${os.arch()}`;
|
|
120
|
-
switch (key) {
|
|
121
|
-
case 'darwin-arm64':
|
|
122
|
-
case 'darwin-x64':
|
|
123
|
-
case 'linux-arm64':
|
|
124
|
-
case 'linux-x64':
|
|
125
|
-
case 'win32-x64':
|
|
126
|
-
// Listed for completeness rather than reach: no engine addon is built for
|
|
127
|
-
// win32-arm64 (`@qvac/llm-llamacpp/prebuilds` has no such target, and no
|
|
128
|
-
// windows-arm runner exists to build one), so nothing on that platform can
|
|
129
|
-
// load a model today. Calibrating it needs that build first, not a fixture.
|
|
130
|
-
case 'win32-arm64':
|
|
131
|
-
case 'android-arm64':
|
|
132
|
-
case 'ios-arm64':
|
|
133
|
-
return key;
|
|
134
|
-
default:
|
|
135
|
-
return undefined;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
@@ -2,6 +2,7 @@ import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
|
|
|
2
2
|
import { aggregateEvents } from '../utils/aggregate-events.js';
|
|
3
3
|
import { CompletionFailedError } from '../errors/index.js';
|
|
4
4
|
import { dispatchPluginStream } from './plugin-dispatch.js';
|
|
5
|
+
import { executeToolSearch, TOOL_SEARCH_NAME } from '../utils/tools/defer.js';
|
|
5
6
|
const DEFAULT_MAX_TOOL_TURNS = 8;
|
|
6
7
|
// Generous ceiling on one client-side tool execution: long enough for a slow
|
|
7
8
|
// MCP round trip, short enough that a client that stopped reading can't pin a
|
|
@@ -154,7 +155,23 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
|
|
|
154
155
|
// The assistant's tool-call turn goes back verbatim so the model sees
|
|
155
156
|
// its own call syntax; each result follows as a `tool` message.
|
|
156
157
|
history = [...history, { role: 'assistant', content: rawFullText ?? contentText }];
|
|
158
|
+
// Without a deferred tool, a `tool_search` call is the caller's own tool.
|
|
159
|
+
const defers = request.tools?.some((tool) => tool.deferLoading === true) ?? false;
|
|
157
160
|
for (const call of toolCalls) {
|
|
161
|
+
// `tool_search` is the SDK's own tool: it reads the registered inventory
|
|
162
|
+
// and appends the matched definitions, so it never leaves the worker and
|
|
163
|
+
// the client is never asked to run it. The `toolCall` event still went
|
|
164
|
+
// downstream with the turn's events, so the client can see it happened.
|
|
165
|
+
if (defers && call.name === TOOL_SEARCH_NAME) {
|
|
166
|
+
history = [
|
|
167
|
+
...history,
|
|
168
|
+
{
|
|
169
|
+
role: 'tool',
|
|
170
|
+
content: executeToolSearch(request.tools ?? [], call.arguments, history)
|
|
171
|
+
}
|
|
172
|
+
];
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
158
175
|
yield {
|
|
159
176
|
type: 'completionOrchestrate',
|
|
160
177
|
turn,
|
|
@@ -15,6 +15,8 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
15
15
|
logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
|
|
16
16
|
}
|
|
17
17
|
const handlers = plugin ? Object.keys(plugin.handlers) : [];
|
|
18
|
+
const fitProbe = entry.local.fitProbe;
|
|
19
|
+
const probeRan = fitProbe !== undefined && fitProbe.reason !== 'disabled';
|
|
18
20
|
const info = {
|
|
19
21
|
modelId: entry.id,
|
|
20
22
|
modelType: entry.local.modelType,
|
|
@@ -27,7 +29,8 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
27
29
|
// Same detection the completion normalizer uses, so callers see the dialect it parses.
|
|
28
30
|
...(handlers.includes('completionStream') && {
|
|
29
31
|
toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
|
|
30
|
-
})
|
|
32
|
+
}),
|
|
33
|
+
...(probeRan && { fitProbe })
|
|
31
34
|
};
|
|
32
35
|
return { type: 'getLoadedModelInfo', info };
|
|
33
36
|
}
|
|
@@ -5,6 +5,9 @@ import { promises as fsPromises } from 'bare-fs';
|
|
|
5
5
|
import { getShardPath, getModelsCacheDir, getSingleFileCachePath } from '../utils/cache/paths.js';
|
|
6
6
|
import { validateAndJoinPath } from '../utils/path-security.js';
|
|
7
7
|
import { ModelNotFoundError } from '../errors/index.js';
|
|
8
|
+
import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, getCoremlSetKey } from './load-model/coreml-sidecars.js';
|
|
9
|
+
import { getRegistryClient } from '../runtime/registry-client.js';
|
|
10
|
+
import { getRuntimeContext } from '../runtime/state.js';
|
|
8
11
|
export async function handleGetModelInfo(request) {
|
|
9
12
|
const { name } = request;
|
|
10
13
|
const catalogEntry = models.find((m) => m.name === name);
|
|
@@ -16,7 +19,7 @@ export async function handleGetModelInfo(request) {
|
|
|
16
19
|
? await handleShardedModel(cacheKey, catalogEntry.shardMetadata)
|
|
17
20
|
: catalogEntry.companionSet
|
|
18
21
|
? await handleCompanionSetModel(catalogEntry.companionSet, catalogEntry.registryPath, catalogEntry.addon)
|
|
19
|
-
: await
|
|
22
|
+
: await handleOptionalCoremlModel(catalogEntry);
|
|
20
23
|
const { cacheFiles, isCached, actualSize, cachedAt, primaryPath } = cacheStatus;
|
|
21
24
|
const loadedModelIds = getAllModelIds();
|
|
22
25
|
const matchPath = primaryPath ?? cacheFiles[0]?.path;
|
|
@@ -64,6 +67,31 @@ export async function handleGetModelInfo(request) {
|
|
|
64
67
|
modelInfo
|
|
65
68
|
};
|
|
66
69
|
}
|
|
70
|
+
async function handleOptionalCoremlModel(model) {
|
|
71
|
+
const platform = getRuntimeContext().platform;
|
|
72
|
+
const spec = getCoremlSidecarSpec(model.registryPath, model.registrySource, platform);
|
|
73
|
+
if (spec) {
|
|
74
|
+
const cachedSet = await findLocallyCachedCoremlCompanionSet(model, platform);
|
|
75
|
+
if (cachedSet) {
|
|
76
|
+
const cachedStatus = await handleCompanionSetModel(cachedSet, model.registryPath, model.addon);
|
|
77
|
+
if (cachedStatus.isCached)
|
|
78
|
+
return cachedStatus;
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const filename = model.registryPath.split('/').pop() || model.registryPath;
|
|
82
|
+
const primaryPath = validateAndJoinPath(getModelsCacheDir(), 'sets', getCoremlSetKey(model, spec), filename);
|
|
83
|
+
await fsPromises.access(primaryPath);
|
|
84
|
+
const set = await findCoremlCompanionSet(await getRegistryClient(), model, platform);
|
|
85
|
+
if (set) {
|
|
86
|
+
const status = await handleCompanionSetModel(set, model.registryPath, model.addon);
|
|
87
|
+
if (status.isCached)
|
|
88
|
+
return status;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch { }
|
|
92
|
+
}
|
|
93
|
+
return handleSingleFileModel(model.registryPath, model.expectedSize, model.sha256Checksum);
|
|
94
|
+
}
|
|
67
95
|
async function handleShardedModel(cacheKey, shardMetadata) {
|
|
68
96
|
const fileEntries = shardMetadata.map((s) => ({
|
|
69
97
|
filename: s.filename,
|