@qvac/inference 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +50 -25
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
package/dist/api/video.d.ts
CHANGED
|
@@ -16,13 +16,14 @@ export interface VideoResult {
|
|
|
16
16
|
* @param params - Video request parameters (model, prompt, dimensions, frame count, fps, sampler, seed, etc.).
|
|
17
17
|
* @returns A result object exposing `requestId` (stable identifier for this in-flight generation), `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated video buffers, typically a single AVI file), and `stats` (promise of generation statistics).
|
|
18
18
|
*
|
|
19
|
-
* Supports `txt2vid` (text-to-video) and `img2vid` (image-to-video) for
|
|
20
|
-
*
|
|
19
|
+
* Supports `txt2vid` (text-to-video) and `img2vid` (image-to-video) for the Wan,
|
|
20
|
+
* LTX-2, and MiniMax-H3 layouts. For Wan `img2vid`, load the pipeline with
|
|
21
21
|
* `modelConfig.clipVisionModelSrc` set to `clip_vision_h.safetensors`; LTX-2
|
|
22
22
|
* `img2vid` conditions on the first frame through its video VAE and needs no
|
|
23
23
|
* CLIP vision weights (the same LTX-2 model loaded for txt2vid also does
|
|
24
|
-
* img2vid).
|
|
25
|
-
* for
|
|
24
|
+
* img2vid). MiniMax-H3 also conditions through its video VAE and can optionally
|
|
25
|
+
* load `modelConfig.audioVaeModelSrc` for synchronized audio. On React Native,
|
|
26
|
+
* the bundled video diffusion models are too large for typical mobile devices.
|
|
26
27
|
*
|
|
27
28
|
* @example Basic txt2vid generation
|
|
28
29
|
* ```typescript
|
package/dist/api/video.js
CHANGED
|
@@ -9,13 +9,14 @@ import { decodeBase64 } from '../utils/encoding.js';
|
|
|
9
9
|
* @param params - Video request parameters (model, prompt, dimensions, frame count, fps, sampler, seed, etc.).
|
|
10
10
|
* @returns A result object exposing `requestId` (stable identifier for this in-flight generation), `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated video buffers, typically a single AVI file), and `stats` (promise of generation statistics).
|
|
11
11
|
*
|
|
12
|
-
* Supports `txt2vid` (text-to-video) and `img2vid` (image-to-video) for
|
|
13
|
-
*
|
|
12
|
+
* Supports `txt2vid` (text-to-video) and `img2vid` (image-to-video) for the Wan,
|
|
13
|
+
* LTX-2, and MiniMax-H3 layouts. For Wan `img2vid`, load the pipeline with
|
|
14
14
|
* `modelConfig.clipVisionModelSrc` set to `clip_vision_h.safetensors`; LTX-2
|
|
15
15
|
* `img2vid` conditions on the first frame through its video VAE and needs no
|
|
16
16
|
* CLIP vision weights (the same LTX-2 model loaded for txt2vid also does
|
|
17
|
-
* img2vid).
|
|
18
|
-
* for
|
|
17
|
+
* img2vid). MiniMax-H3 also conditions through its video VAE and can optionally
|
|
18
|
+
* load `modelConfig.audioVaeModelSrc` for synchronized audio. On React Native,
|
|
19
|
+
* the bundled video diffusion models are too large for typical mobile devices.
|
|
19
20
|
*
|
|
20
21
|
* @example Basic txt2vid generation
|
|
21
22
|
* ```typescript
|
package/dist/errors/index.d.ts
CHANGED
|
@@ -66,6 +66,9 @@ export declare class ModelFileNotFoundError extends QvacErrorBase {
|
|
|
66
66
|
export declare class ModelFileNotFoundInDirError extends QvacErrorBase {
|
|
67
67
|
constructor(modelFile: string, modelDir: string, modelType: string, cause?: unknown);
|
|
68
68
|
}
|
|
69
|
+
export declare class ModelFitRefusedError extends QvacErrorBase {
|
|
70
|
+
constructor(modelType: string, reason: string, detail?: string, cause?: unknown);
|
|
71
|
+
}
|
|
69
72
|
export declare class ModelFileLocateFailedError extends QvacErrorBase {
|
|
70
73
|
constructor(modelType: string, modelPath: string, cause?: unknown);
|
|
71
74
|
}
|
package/dist/errors/index.js
CHANGED
|
@@ -117,6 +117,11 @@ export class ModelFileNotFoundInDirError extends QvacErrorBase {
|
|
|
117
117
|
super(createErrorOptions(ERROR_CODES.MODEL_FILE_NOT_FOUND_IN_DIR, [modelFile, modelDir, modelType], cause));
|
|
118
118
|
}
|
|
119
119
|
}
|
|
120
|
+
export class ModelFitRefusedError extends QvacErrorBase {
|
|
121
|
+
constructor(modelType, reason, detail, cause) {
|
|
122
|
+
super(createErrorOptions(ERROR_CODES.MODEL_FIT_REFUSED, detail === undefined ? [modelType, reason] : [modelType, reason, detail], cause));
|
|
123
|
+
}
|
|
124
|
+
}
|
|
120
125
|
export class ModelFileLocateFailedError extends QvacErrorBase {
|
|
121
126
|
constructor(modelType, modelPath, cause) {
|
|
122
127
|
super(createErrorOptions(ERROR_CODES.MODEL_FILE_LOCATE_FAILED, [modelType, modelPath], cause));
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate, ModelFitEstimateTarget } from '../schemas/assess-model-fit.js';
|
|
2
|
+
import type { FitStubUnavailableReason } from '../resources/model-fit/fit-stub/fetch-fit-stub.js';
|
|
2
3
|
/**
|
|
3
4
|
* Runs a pre-download fit assessment worker-side.
|
|
4
5
|
*
|
|
5
6
|
* This lives on the worker because that is where the three things it needs
|
|
6
7
|
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
7
8
|
* own platform/arch pair, and the registry client. No weights are read and
|
|
8
|
-
* nothing is loaded —
|
|
9
|
-
*
|
|
10
|
-
* answer instead of the coefficients modelling it.
|
|
9
|
+
* nothing is loaded — each candidate additionally has the registry's weightless
|
|
10
|
+
* description fetched, tens of KB, so the engine's own fitter can answer.
|
|
11
11
|
*/
|
|
12
12
|
export declare function handleAssessModelFit(request: AssessModelFitRequest): Promise<AssessModelFitResponse>;
|
|
13
13
|
/**
|
|
@@ -20,3 +20,5 @@ export declare function handleAssessModelFit(request: AssessModelFitRequest): Pr
|
|
|
20
20
|
* type stands in as the label.
|
|
21
21
|
*/
|
|
22
22
|
export declare function estimateTargetFor(candidate: ModelFitCandidate): ModelFitEstimateTarget;
|
|
23
|
+
/** Why a load with no description staged for it has no engine verdict. */
|
|
24
|
+
export declare function noStubDetail(reason: FitStubUnavailableReason, message?: string): string;
|
|
@@ -5,7 +5,6 @@ import { projectFitFromLoad } from '../resources/model-fit/fit-stub/project-fit-
|
|
|
5
5
|
import { getResourceCollector } from '../resources/instance.js';
|
|
6
6
|
import { getConfig } from '../runtime/state.js';
|
|
7
7
|
import { assessModelFitFromResources } from '../resources/model-fit/assess.js';
|
|
8
|
-
import { getPlatformCalibration } from '../resources/model-fit/calibration/index.js';
|
|
9
8
|
import { detectPlatform } from '../resources/model-fit/platform.js';
|
|
10
9
|
/**
|
|
11
10
|
* Runs a pre-download fit assessment worker-side.
|
|
@@ -13,21 +12,18 @@ import { detectPlatform } from '../resources/model-fit/platform.js';
|
|
|
13
12
|
* This lives on the worker because that is where the three things it needs
|
|
14
13
|
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
15
14
|
* own platform/arch pair, and the registry client. No weights are read and
|
|
16
|
-
* nothing is loaded —
|
|
17
|
-
*
|
|
18
|
-
* answer instead of the coefficients modelling it.
|
|
15
|
+
* nothing is loaded — each candidate additionally has the registry's weightless
|
|
16
|
+
* description fetched, tens of KB, so the engine's own fitter can answer.
|
|
19
17
|
*/
|
|
20
18
|
export async function handleAssessModelFit(request) {
|
|
21
19
|
const platform = detectPlatform();
|
|
22
|
-
const
|
|
20
|
+
const nativeFits = await resolveNativeFits(request.models);
|
|
23
21
|
const result = assessModelFitFromResources({
|
|
24
22
|
models: request.models.map(estimateTargetFor),
|
|
25
23
|
execution: request.execution,
|
|
26
24
|
resources: readResources(),
|
|
27
25
|
platform,
|
|
28
|
-
|
|
29
|
-
nativeFit: native.fit,
|
|
30
|
-
nativeFitUnavailable: native.unavailable
|
|
26
|
+
nativeFits
|
|
31
27
|
});
|
|
32
28
|
return { type: 'assessModelFit', ...result };
|
|
33
29
|
}
|
|
@@ -47,6 +43,7 @@ function modelTypeOf(candidate) {
|
|
|
47
43
|
}
|
|
48
44
|
/** The audio engines, whose estimator sizes a load by its window rather than a context. */
|
|
49
45
|
const AUDIO_ENGINES = [
|
|
46
|
+
ModelType.mossTranscribe,
|
|
50
47
|
'whispercpp-transcription',
|
|
51
48
|
'parakeet-transcription',
|
|
52
49
|
'bci-whispercpp-transcription'
|
|
@@ -62,17 +59,30 @@ function contextUsesGpu(config) {
|
|
|
62
59
|
return undefined;
|
|
63
60
|
return contextParams['use_gpu'];
|
|
64
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* MiniMax names its device instead of switching the GPU on, and the config
|
|
64
|
+
* schema refuses both at once. `auto` is what the switch means: the GPU when
|
|
65
|
+
* one is usable.
|
|
66
|
+
*/
|
|
67
|
+
function audiogenUsesGpu(config) {
|
|
68
|
+
const device = config['device'];
|
|
69
|
+
if (device !== undefined)
|
|
70
|
+
return device === 'gpu' || device === 'auto';
|
|
71
|
+
return config['useGPU'] === true;
|
|
72
|
+
}
|
|
65
73
|
/**
|
|
66
74
|
* Engines that carry a GPU switch, each read at the key its own config spells
|
|
67
75
|
* it under. These are the keys the fit builders read in `native-probe/engines`,
|
|
68
76
|
* so a projection and a device never disagree. Only bci defaults to on.
|
|
69
77
|
*/
|
|
70
78
|
const GPU_SWITCH = {
|
|
79
|
+
[ModelType.mossTranscribe]: (config) => config['useGPU'] === true,
|
|
71
80
|
[ModelType.ttsGgml]: (config) => config['useGPU'] === true,
|
|
72
|
-
[ModelType.audiogenGgml]:
|
|
81
|
+
[ModelType.audiogenGgml]: audiogenUsesGpu,
|
|
73
82
|
[ModelType.parakeetTranscription]: (config) => config['useGPU'] === true,
|
|
74
83
|
[ModelType.whispercppTranscription]: (config) => contextUsesGpu(config) === true,
|
|
75
|
-
[ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false
|
|
84
|
+
[ModelType.bciWhispercppTranscription]: (config) => contextUsesGpu(config) !== false,
|
|
85
|
+
[ModelType.ggmlOcr]: (config) => (config['backendDevice'] ?? 'cpu') !== 'cpu'
|
|
76
86
|
};
|
|
77
87
|
/**
|
|
78
88
|
* Where a load resolved to run. `dispatch` applies the host's device defaults
|
|
@@ -174,6 +184,11 @@ export function estimateTargetFor(candidate) {
|
|
|
174
184
|
const contextTokens = config['ctx_size'];
|
|
175
185
|
const artifacts = companionRefs(config);
|
|
176
186
|
const device = resolvedDevice(modelType, config);
|
|
187
|
+
const windowMs = modelType === ModelType.mossTranscribe
|
|
188
|
+
? candidate.transcriptionWorkload === undefined
|
|
189
|
+
? undefined
|
|
190
|
+
: candidate.transcriptionWorkload.audioSeconds * 1000
|
|
191
|
+
: audioWindowMs(config);
|
|
177
192
|
return {
|
|
178
193
|
model: {
|
|
179
194
|
name: descriptor?.name ?? descriptor?.modelId ?? location ?? modelType,
|
|
@@ -188,7 +203,7 @@ export function estimateTargetFor(candidate) {
|
|
|
188
203
|
workload: AUDIO_ENGINES.includes(modelType)
|
|
189
204
|
? {
|
|
190
205
|
kind: 'audio',
|
|
191
|
-
windowMs
|
|
206
|
+
...(windowMs !== undefined && { windowMs }),
|
|
192
207
|
streaming: config['streaming'] === true
|
|
193
208
|
}
|
|
194
209
|
: {
|
|
@@ -197,17 +212,8 @@ export function estimateTargetFor(candidate) {
|
|
|
197
212
|
}
|
|
198
213
|
};
|
|
199
214
|
}
|
|
200
|
-
/**
|
|
201
|
-
|
|
202
|
-
* load's own plugin. Only for a one-candidate request: the probe measures one
|
|
203
|
-
* model against the whole machine, which cannot be aggregated across a set.
|
|
204
|
-
*/
|
|
205
|
-
async function resolveNativeFit(candidates) {
|
|
206
|
-
if (candidates.length !== 1)
|
|
207
|
-
return {};
|
|
208
|
-
const candidate = candidates[0];
|
|
209
|
-
if (!candidate)
|
|
210
|
-
return {};
|
|
215
|
+
/** The engine fitter's verdict for one candidate, through that load's plugin. */
|
|
216
|
+
async function resolveNativeFit(candidate, alreadyCounted) {
|
|
211
217
|
const modelType = modelTypeOf(candidate);
|
|
212
218
|
if (!isCanonicalModelType(modelType)) {
|
|
213
219
|
return { unavailable: `no plugin handles model type ${modelType}` };
|
|
@@ -215,18 +221,64 @@ async function resolveNativeFit(candidates) {
|
|
|
215
221
|
const budgetMs = getConfig().fitStubBudgetMs;
|
|
216
222
|
const outcome = await projectFitFromLoad({
|
|
217
223
|
modelType,
|
|
224
|
+
...(candidate.transcriptionWorkload !== undefined && {
|
|
225
|
+
transcriptionWorkload: candidate.transcriptionWorkload
|
|
226
|
+
}),
|
|
227
|
+
...(candidate.ttsWorkload !== undefined && { ttsWorkload: candidate.ttsWorkload }),
|
|
218
228
|
...(candidate.modelSrc !== undefined && { modelSrc: candidate.modelSrc }),
|
|
219
229
|
...(candidate.modelConfig !== undefined && { modelConfig: candidate.modelConfig })
|
|
220
|
-
}, estimateTargetFor(candidate).model.name,
|
|
230
|
+
}, estimateTargetFor(candidate).model.name, {
|
|
231
|
+
...(budgetMs !== undefined && { stub: { budgetMs } }),
|
|
232
|
+
...(alreadyCounted > 0 && { fit: { extraResidentBytes: alreadyCounted } })
|
|
233
|
+
});
|
|
221
234
|
if (outcome.status === 'projected')
|
|
222
235
|
return { fit: outcome.fit };
|
|
223
236
|
if (outcome.status === 'unsupported-load')
|
|
224
237
|
return { unavailable: outcome.detail };
|
|
225
|
-
return {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
238
|
+
return { unavailable: noStubDetail(outcome.reason, outcome.message) };
|
|
239
|
+
}
|
|
240
|
+
/** Why a load with no description staged for it has no engine verdict. */
|
|
241
|
+
export function noStubDetail(reason, message) {
|
|
242
|
+
if (reason === 'files-only-engine') {
|
|
243
|
+
return 'the engine measures downloaded files only, and this model is not downloaded';
|
|
244
|
+
}
|
|
245
|
+
return message === undefined
|
|
246
|
+
? `no registry description (${reason})`
|
|
247
|
+
: `no registry description (${reason}): ${message}`;
|
|
248
|
+
}
|
|
249
|
+
/** What a projection holds for the model's lifetime, the breakdown or the total. */
|
|
250
|
+
function residentBytesOf(fit) {
|
|
251
|
+
const projection = fit.fit?.projection;
|
|
252
|
+
if (!projection)
|
|
253
|
+
return 0;
|
|
254
|
+
const { weightsBytes, contextBytes, deviceBytes, hostBytes } = projection;
|
|
255
|
+
const resident = weightsBytes === undefined || contextBytes === undefined
|
|
256
|
+
? deviceBytes
|
|
257
|
+
: weightsBytes + contextBytes;
|
|
258
|
+
return (resident ?? 0) + (hostBytes ?? 0);
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* One verdict per candidate, in request order. Each probe measures its own
|
|
262
|
+
* model against the whole machine, so the bytes compose where the verdicts do
|
|
263
|
+
* not, and `assess` combines them under one budget.
|
|
264
|
+
*
|
|
265
|
+
* Run one at a time, so a set never holds several fitters open at once.
|
|
266
|
+
*
|
|
267
|
+
* Each probe holds back what the earlier ones projected, since none of them is
|
|
268
|
+
* registered in this worker and the fitter would otherwise place every
|
|
269
|
+
* candidate in the same free memory. Every model is resident under either
|
|
270
|
+
* execution mode, so the holdback does not read it. The answer follows the
|
|
271
|
+
* order the caller listed the models in.
|
|
272
|
+
*/
|
|
273
|
+
async function resolveNativeFits(candidates) {
|
|
274
|
+
const fits = [];
|
|
275
|
+
let alreadyCounted = 0;
|
|
276
|
+
for (const candidate of candidates) {
|
|
277
|
+
const fit = await resolveNativeFit(candidate, alreadyCounted);
|
|
278
|
+
fits.push(fit);
|
|
279
|
+
alreadyCounted += residentBytesOf(fit);
|
|
280
|
+
}
|
|
281
|
+
return fits;
|
|
230
282
|
}
|
|
231
283
|
function readResources() {
|
|
232
284
|
const collector = getResourceCollector();
|
|
@@ -16,7 +16,6 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
16
16
|
}
|
|
17
17
|
const handlers = plugin ? Object.keys(plugin.handlers) : [];
|
|
18
18
|
const fitProbe = entry.local.fitProbe;
|
|
19
|
-
const probeRan = fitProbe !== undefined && fitProbe.reason !== 'disabled';
|
|
20
19
|
const info = {
|
|
21
20
|
modelId: entry.id,
|
|
22
21
|
modelType: entry.local.modelType,
|
|
@@ -30,7 +29,7 @@ export function handleGetLoadedModelInfo(request) {
|
|
|
30
29
|
...(handlers.includes('completionStream') && {
|
|
31
30
|
toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
|
|
32
31
|
}),
|
|
33
|
-
...(
|
|
32
|
+
...(fitProbe !== undefined && { fitProbe })
|
|
34
33
|
};
|
|
35
34
|
return { type: 'getLoadedModelInfo', info };
|
|
36
35
|
}
|
|
@@ -6,9 +6,6 @@ import { getEngineLogger } from '../../logging/index.js';
|
|
|
6
6
|
const logger = getEngineLogger();
|
|
7
7
|
const MAX_REDIRECTS = 20;
|
|
8
8
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
9
|
-
function protocolFor(url) {
|
|
10
|
-
return url.protocol === 'https:' ? https : http;
|
|
11
|
-
}
|
|
12
9
|
function assertSecureHop(url, previous) {
|
|
13
10
|
if (isSecureDownloadUrl(url))
|
|
14
11
|
return;
|
|
@@ -22,7 +19,6 @@ function assertSecureHop(url, previous) {
|
|
|
22
19
|
// connect can't leak.
|
|
23
20
|
function requestOnce(url, method, headers, timeoutMs, signal) {
|
|
24
21
|
return new Promise((resolve, reject) => {
|
|
25
|
-
const protocol = protocolFor(url);
|
|
26
22
|
let settled = false;
|
|
27
23
|
let timer;
|
|
28
24
|
const cleanup = () => {
|
|
@@ -45,7 +41,7 @@ function requestOnce(url, method, headers, timeoutMs, signal) {
|
|
|
45
41
|
reject(error);
|
|
46
42
|
};
|
|
47
43
|
const onAbort = () => fail(new DownloadCancelledError());
|
|
48
|
-
const
|
|
44
|
+
const onResponse = (res) => {
|
|
49
45
|
if (settled) {
|
|
50
46
|
res.destroy();
|
|
51
47
|
return;
|
|
@@ -53,7 +49,10 @@ function requestOnce(url, method, headers, timeoutMs, signal) {
|
|
|
53
49
|
settled = true;
|
|
54
50
|
cleanup();
|
|
55
51
|
resolve(res);
|
|
56
|
-
}
|
|
52
|
+
};
|
|
53
|
+
const req = url.protocol === 'https:'
|
|
54
|
+
? https.request(url, { method, headers }, (res) => onResponse(res))
|
|
55
|
+
: http.request(url, { method, headers }, onResponse);
|
|
57
56
|
req.on('error', (error) => fail(error));
|
|
58
57
|
if (signal) {
|
|
59
58
|
if (signal.aborted) {
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import type { ModelRegistryListResponse, ModelRegistrySearchRequest, ModelRegistrySearchResponse, ModelRegistryGetModelRequest, ModelRegistryGetModelResponse } from '../schemas/index.js';
|
|
1
|
+
import type { ModelRegistryListResponse, ModelRegistrySearchRequest, ModelRegistrySearchResponse, ModelRegistryGetModelRequest, ModelRegistryGetModelResponse, ModelRegistryEntry } from '../schemas/index.js';
|
|
2
|
+
import type { QVACModelEntry } from '@qvac/registry-client';
|
|
3
|
+
export declare function processRegistryModel(model: QVACModelEntry): ModelRegistryEntry | null;
|
|
2
4
|
export declare function handleModelRegistryList(): Promise<ModelRegistryListResponse>;
|
|
3
5
|
export declare function handleModelRegistrySearch(request: ModelRegistrySearchRequest): Promise<ModelRegistrySearchResponse>;
|
|
4
6
|
export declare function handleModelRegistryGetModel(request: ModelRegistryGetModelRequest): Promise<ModelRegistryGetModelResponse>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import Buffer from 'bare-buffer';
|
|
2
2
|
import { REGISTRY_ERROR_CODES } from '../schemas/errors.js';
|
|
3
|
-
import { getAddonFromEngine, resolveCanonicalEngine } from '../schemas/engine-addon-map.js';
|
|
3
|
+
import { getAddonFromEngine, resolveCanonicalEngine, resolveRegistryEngine } from '../schemas/engine-addon-map.js';
|
|
4
4
|
import { getRegistryClient } from '../runtime/registry-client.js';
|
|
5
5
|
import { getEngineLogger } from '../logging/index.js';
|
|
6
6
|
import { ModelRegistryQueryFailedError } from '../errors/index.js';
|
|
@@ -17,8 +17,8 @@ function toHexString(value) {
|
|
|
17
17
|
}
|
|
18
18
|
return '';
|
|
19
19
|
}
|
|
20
|
-
function processRegistryModel(model) {
|
|
21
|
-
const engine =
|
|
20
|
+
export function processRegistryModel(model) {
|
|
21
|
+
const engine = resolveRegistryEngine(model.engine || '', model.tags);
|
|
22
22
|
if (!engine) {
|
|
23
23
|
logger.warn(`Skipping model with unknown engine "${model.engine}": ${model.path}`);
|
|
24
24
|
return null;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { registerRpcServerProvider } from './rpc/provider.js';
|
|
2
|
-
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, type VectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, startRpcServer, stopRpcServer, discoverRpcServers, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, audioGen, audioEdit, audioUnderstand, classify, video, type VideoProgressTick, upscale, worldCreateScene, worldStep, type WorldStepResult, type WorldStepProgressTick, type WorldSceneResult, type WorldSceneResultWithPack, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle } from './api/index.js';
|
|
2
|
+
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, type VectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, startRpcServer, stopRpcServer, discoverRpcServers, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, audioGen, audioEdit, audioUnderstand, classify, decide, video, type VideoProgressTick, upscale, worldCreateScene, worldStep, type WorldStepResult, type WorldStepProgressTick, type WorldSceneResult, type WorldSceneResultWithPack, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle } from './api/index.js';
|
|
3
3
|
export { close } from './dispatch.js';
|
|
4
4
|
export { plugins, registerPlugin, registerPlugins, getPlugin, hasPlugin, getAllPlugins, unregisterPlugin } from './plugins/index.js';
|
|
5
5
|
export * from './surface.js';
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@ export { registerRpcServerProvider } from './rpc/provider.js';
|
|
|
3
3
|
// Public API: the operations, engine lifecycle and explicit plugin assembly, and
|
|
4
4
|
// the value-clean surface (schemas / types / consts / errors / model registry)
|
|
5
5
|
// re-exported from `./surface`.
|
|
6
|
-
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, startRpcServer, stopRpcServer, discoverRpcServers, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, audioGen, audioEdit, audioUnderstand, classify, video, upscale, worldCreateScene, worldStep, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './api/index.js';
|
|
6
|
+
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, createVectorIndex, loadVectorIndex, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, startRpcServer, stopRpcServer, discoverRpcServers, getSystemResources, assessModelFit, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, audioGen, audioEdit, audioUnderstand, classify, decide, video, upscale, worldCreateScene, worldStep, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './api/index.js';
|
|
7
7
|
// Engine lifecycle and explicit plugin assembly. No plugins are registered by
|
|
8
8
|
// default: an app assembles the engines it needs via `plugins([...])` or the
|
|
9
9
|
// `registerPlugin`/`registerPlugins` primitives, then calls the operations.
|
package/dist/logging/logger.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import QvacLogger from '@qvac/logging';
|
|
2
|
-
import { LOG_LEVELS } from '@qvac/logging/constants';
|
|
2
|
+
import { LOG_LEVELS } from '@qvac/logging/constants.js';
|
|
3
3
|
import { sendLogToStreams } from '../runtime/logging-stream-registry.js';
|
|
4
4
|
import { isLevelEnabled, formatArg } from './utils.js';
|
|
5
5
|
import { LOG_ID, LOG_NAMESPACE } from './namespaces.js';
|
package/dist/logging/utils.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LEVEL_PRIORITIES, LOG_LEVELS } from '@qvac/logging/constants';
|
|
1
|
+
import { LEVEL_PRIORITIES, LOG_LEVELS } from '@qvac/logging/constants.js';
|
|
2
2
|
import stringify from 'fast-safe-stringify';
|
|
3
3
|
export function isLevelEnabled(messageLevel, currentLevel) {
|
|
4
4
|
// "off" suppresses every emission (console, streams, and transports alike),
|