@qvac/inference 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +30 -1
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
|
@@ -1,120 +1,6 @@
|
|
|
1
|
-
import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
-
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
3
|
-
/** A memory bound. `lower` is the optimistic end, `upper` the conservative one. */
|
|
4
|
-
export interface ByteRange {
|
|
5
|
-
lower: number;
|
|
6
|
-
upper: number;
|
|
7
|
-
}
|
|
8
1
|
/**
|
|
9
|
-
* Platforms
|
|
10
|
-
*
|
|
2
|
+
* Platforms this assessment covers. A platform absent from this list assesses
|
|
3
|
+
* as `unknown`, because nothing fixes the memory budget it would be judged
|
|
4
|
+
* against.
|
|
11
5
|
*/
|
|
12
6
|
export type ModelFitPlatform = 'darwin-arm64' | 'darwin-x64' | 'linux-arm64' | 'linux-x64' | 'win32-x64' | 'win32-arm64' | 'android-arm64' | 'ios-arm64';
|
|
13
|
-
/**
|
|
14
|
-
* Measured coefficients for one platform. Derived by the calibration harness
|
|
15
|
-
* (`scripts/calibrate-model-fit.ts`) from real loads, then committed.
|
|
16
|
-
*
|
|
17
|
-
* @property weightUpperCoeff - Multiplier on `artifactBytes` for resident
|
|
18
|
-
* weights at the conservative end. The lower end is always 1.0: llama.cpp
|
|
19
|
-
* maps weights, so the file size is the floor.
|
|
20
|
-
* @property fixedOverheadBytes - Runtime cost that does not scale with the
|
|
21
|
-
* workload: engine context, backend buffers, tokenizer.
|
|
22
|
-
* @property computeBufferBytesPerToken - Compute/graph buffers that scale with
|
|
23
|
-
* the context the caller asks for.
|
|
24
|
-
* @property audioWindowBytes - Whisper-family working memory for one 30 s
|
|
25
|
-
* window, the engine's own default window.
|
|
26
|
-
* @property audioStreamingBytes - Extra resident cost of a streaming session.
|
|
27
|
-
* @property validated - False while coefficients are provisional. An
|
|
28
|
-
* unvalidated platform assesses as `unknown`, exactly like a missing one.
|
|
29
|
-
* @property measuredOn - What the harness observed while measuring. Recorded
|
|
30
|
-
* because these coefficients are keyed by platform alone, while the buffers
|
|
31
|
-
* they measure are allocated by the backend: a `linux-x64` fixture measured on
|
|
32
|
-
* CUDA is being applied to CPU-only hosts too. Stating each fixture's scope
|
|
33
|
-
* keeps that visible, and means the key could later gain a backend dimension
|
|
34
|
-
* without re-measuring anything. Absent on a placeholder, which measured
|
|
35
|
-
* nothing.
|
|
36
|
-
*/
|
|
37
|
-
export interface PlatformCalibration {
|
|
38
|
-
weightUpperCoeff: number;
|
|
39
|
-
fixedOverheadBytes: ByteRange;
|
|
40
|
-
computeBufferBytesPerToken: ByteRange;
|
|
41
|
-
/**
|
|
42
|
-
* Transient peak one operation adds on top of the resident cost — a few MiB
|
|
43
|
-
* on most platforms, 73 MiB on darwin-x64. Optional: fixtures measured
|
|
44
|
-
* before the harness sampled generation contribute nothing, as they do now.
|
|
45
|
-
*/
|
|
46
|
-
workingPeakBytes?: ByteRange;
|
|
47
|
-
audioWindowBytes: ByteRange;
|
|
48
|
-
audioStreamingBytes: ByteRange;
|
|
49
|
-
validated: boolean;
|
|
50
|
-
measuredAt?: string;
|
|
51
|
-
measuredOn?: CalibrationProvenance;
|
|
52
|
-
notes?: readonly string[];
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* The measurement conditions behind one set of coefficients.
|
|
56
|
-
*
|
|
57
|
-
* @property backend - Best-effort label for the backend in play during the run,
|
|
58
|
-
* inferred from the GPU drivers the resource collector reported: `metal`,
|
|
59
|
-
* `vulkan`, `cuda`, `rocm`, or `cpu`.
|
|
60
|
-
* @property device - GPU name, when one was reported.
|
|
61
|
-
* @property kvElementBytes - Bytes per KV-cache element the residuals were
|
|
62
|
-
* computed against. The single most useful number here: subtract the wrong
|
|
63
|
-
* width and the fit is wrong, so recording it makes a bad run auditable after
|
|
64
|
-
* the fact.
|
|
65
|
-
*/
|
|
66
|
-
export interface CalibrationProvenance {
|
|
67
|
-
backend: string;
|
|
68
|
-
device?: string;
|
|
69
|
-
kvElementBytes: number;
|
|
70
|
-
}
|
|
71
|
-
export interface CalibrationFixture {
|
|
72
|
-
schemaVersion: 1;
|
|
73
|
-
platforms: Partial<Record<ModelFitPlatform, PlatformCalibration>>;
|
|
74
|
-
/**
|
|
75
|
-
* GPU-resident coefficients, keyed `<platform>:<backend>`. Separate from
|
|
76
|
-
* `platforms` because those describe CPU-resident execution in system RAM,
|
|
77
|
-
* and because one platform can run several backends whose buffers differ.
|
|
78
|
-
*/
|
|
79
|
-
gpuPlatforms?: Readonly<Record<string, PlatformCalibration>>;
|
|
80
|
-
/**
|
|
81
|
-
* Integrated-GPU coefficients, keyed the same way. A third map because the
|
|
82
|
-
* buffers are the backend's, as in `gpuPlatforms`, but come out of system
|
|
83
|
-
* RAM, as in `platforms`. Measured by the harness's `--igpu` pass.
|
|
84
|
-
*/
|
|
85
|
-
sharedGpuPlatforms?: Readonly<Record<string, PlatformCalibration>>;
|
|
86
|
-
}
|
|
87
|
-
/** What an estimator is handed for one candidate. */
|
|
88
|
-
export interface EstimatorInput {
|
|
89
|
-
profile: ModelResourceProfile;
|
|
90
|
-
workload: ModelFitWorkload;
|
|
91
|
-
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
92
|
-
extraArtifactBytes: number;
|
|
93
|
-
calibration: PlatformCalibration;
|
|
94
|
-
/**
|
|
95
|
-
* Whether the device reports a GPU. Used only to decide which KV-cache type
|
|
96
|
-
* the engine would default to — never as a memory budget, since GPU memory
|
|
97
|
-
* metrics are `unverified`-scoped by design.
|
|
98
|
-
*/
|
|
99
|
-
hasGpu: boolean;
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* An estimate, or a refusal to estimate.
|
|
103
|
-
*
|
|
104
|
-
* `persistent` stays resident for the model's whole lifetime; `working` is the
|
|
105
|
-
* temporary peak of one operation. Aggregation treats them differently, which
|
|
106
|
-
* is what makes `sequential` cheaper than `concurrent`.
|
|
107
|
-
*/
|
|
108
|
-
export type EstimatorResult = {
|
|
109
|
-
kind: 'estimate';
|
|
110
|
-
estimatorVersion: string;
|
|
111
|
-
persistent: ByteRange;
|
|
112
|
-
working: ByteRange;
|
|
113
|
-
reasons: readonly string[];
|
|
114
|
-
assumptions: readonly string[];
|
|
115
|
-
} | {
|
|
116
|
-
kind: 'unknown';
|
|
117
|
-
estimatorVersion: string;
|
|
118
|
-
reasons: readonly string[];
|
|
119
|
-
assumptions?: readonly string[];
|
|
120
|
-
};
|
package/dist/rpc/inventory.js
CHANGED
|
@@ -4,14 +4,12 @@ import { InferenceCancelledError } from '../errors/index.js';
|
|
|
4
4
|
import { getEngineLogger } from '../logging/index.js';
|
|
5
5
|
import { throwIfAborted } from './network.js';
|
|
6
6
|
// Fabric RPC_PROTO_*_VERSION:
|
|
7
|
-
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/
|
|
7
|
+
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/v11018.2.0/ggml/include/ggml-rpc.h
|
|
8
8
|
// Command IDs and HELLO layouts, rpc_cmd and rpc_msg_hello_*:
|
|
9
|
-
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/
|
|
10
|
-
//
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
{ major: 109, minor: 0 }
|
|
14
|
-
];
|
|
9
|
+
// https://github.com/tetherto/qvac-fabric-llm.cpp/blob/v11018.2.0/ggml/src/ggml-rpc/ggml-rpc.cpp
|
|
10
|
+
// Native inference needs matching protocol majors, so only servers on the bundled
|
|
11
|
+
// Fabric's protocol are listed. Patch versions are compatible.
|
|
12
|
+
const SUPPORTED_VERSIONS = [{ major: 111, minor: 0 }];
|
|
15
13
|
const SUPPORTED_VERSION_LABEL = SUPPORTED_VERSIONS.map(({ major, minor }) => `${major}.${minor}.x`).join(', ');
|
|
16
14
|
const HELLO = 14;
|
|
17
15
|
const HELLO_REQUEST_SIZE = 24;
|
package/dist/runtime/env.d.ts
CHANGED
|
@@ -7,12 +7,10 @@ export declare function initEnv(): void;
|
|
|
7
7
|
*/
|
|
8
8
|
export declare function getEnv(): {
|
|
9
9
|
HOME_DIR: string;
|
|
10
|
-
QVAC_ADVISORY_MODEL_FIT?: string | undefined;
|
|
11
10
|
};
|
|
12
11
|
/**
|
|
13
12
|
* Get the validated env config. Must call initEnv() first.
|
|
14
13
|
*/
|
|
15
14
|
export declare function getValidatedEnv(): {
|
|
16
15
|
HOME_DIR: string;
|
|
17
|
-
QVAC_ADVISORY_MODEL_FIT?: string | undefined;
|
|
18
16
|
};
|
package/dist/runtime/env.js
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
import env from 'bare-env';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
const envSchema = z.object({
|
|
4
|
-
HOME_DIR: z.string()
|
|
5
|
-
/**
|
|
6
|
-
* Operator opt-out for the advisory llama.cpp fit check (on by default).
|
|
7
|
-
* Free-form rather than an enum so a typo cannot fail engine startup; only
|
|
8
|
-
* `0`/`false`/`off`/`no` disable it.
|
|
9
|
-
*/
|
|
10
|
-
QVAC_ADVISORY_MODEL_FIT: z.string().optional()
|
|
4
|
+
HOME_DIR: z.string()
|
|
11
5
|
});
|
|
12
6
|
let validatedEnv = null;
|
|
13
7
|
/**
|
|
@@ -18,8 +12,7 @@ export function initEnv() {
|
|
|
18
12
|
// Snap's HOME can be revision-scoped; SNAP_USER_COMMON is stable.
|
|
19
13
|
env['SNAP_USER_COMMON'] ?? env['HOME'] ?? env['USERPROFILE'] ?? '/tmp';
|
|
20
14
|
let envConfig = {
|
|
21
|
-
HOME_DIR: defaultHomeDir
|
|
22
|
-
QVAC_ADVISORY_MODEL_FIT: env['QVAC_ADVISORY_MODEL_FIT']
|
|
15
|
+
HOME_DIR: defaultHomeDir
|
|
23
16
|
};
|
|
24
17
|
const isBareKit = typeof globalThis.BareKit !== 'undefined';
|
|
25
18
|
if (isBareKit && Bare.argv[0]) {
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
import { ModelType } from '../schemas/index.js';
|
|
2
2
|
import { llmConfigSchema, embedConfigSchema } from '../schemas/llamacpp-config.js';
|
|
3
|
-
import { whisperConfigSchema, parakeetRuntimeConfigSchema } from '../schemas/transcription-config.js';
|
|
3
|
+
import { whisperConfigSchema, parakeetRuntimeConfigSchema, mossTranscribeConfigSchema } from '../schemas/transcription-config.js';
|
|
4
4
|
import { bciConfigSchema } from '../schemas/bci-config.js';
|
|
5
5
|
import { ocrConfigSchema } from '../schemas/ocr.js';
|
|
6
6
|
import { sdcppConfigSchema } from '../schemas/sdcpp-config.js';
|
|
7
7
|
import { vlaConfigSchema } from '../schemas/vla.js';
|
|
8
|
+
import { decisionConfigSchema } from '../schemas/decision.js';
|
|
8
9
|
import { classificationConfigSchema } from '../schemas/classification.js';
|
|
9
10
|
import { audioGenConfigSchema } from '../schemas/audio-gen.js';
|
|
10
11
|
export const CANONICAL_TO_ALIAS = {
|
|
11
12
|
[ModelType.llamacppCompletion]: 'llm',
|
|
12
13
|
[ModelType.llamacppEmbedding]: 'embeddings',
|
|
14
|
+
[ModelType.llamacppDecision]: 'llamacpp-decision',
|
|
13
15
|
[ModelType.whispercppTranscription]: 'whisper',
|
|
14
16
|
[ModelType.bciWhispercppTranscription]: 'bci',
|
|
15
17
|
[ModelType.parakeetTranscription]: 'parakeet',
|
|
18
|
+
[ModelType.mossTranscribe]: 'moss-transcribe',
|
|
16
19
|
[ModelType.nmtcppTranslation]: 'nmt',
|
|
17
20
|
[ModelType.onnxTts]: 'tts',
|
|
18
21
|
[ModelType.ttsGgml]: 'tts',
|
|
@@ -25,9 +28,11 @@ export const CANONICAL_TO_ALIAS = {
|
|
|
25
28
|
export const MODEL_CONFIG_SCHEMAS = {
|
|
26
29
|
[ModelType.llamacppCompletion]: llmConfigSchema,
|
|
27
30
|
[ModelType.llamacppEmbedding]: embedConfigSchema,
|
|
31
|
+
[ModelType.llamacppDecision]: decisionConfigSchema,
|
|
28
32
|
[ModelType.whispercppTranscription]: whisperConfigSchema,
|
|
29
33
|
[ModelType.bciWhispercppTranscription]: bciConfigSchema,
|
|
30
34
|
[ModelType.parakeetTranscription]: parakeetRuntimeConfigSchema.passthrough(),
|
|
35
|
+
[ModelType.mossTranscribe]: mossTranscribeConfigSchema,
|
|
31
36
|
[ModelType.ggmlOcr]: ocrConfigSchema,
|
|
32
37
|
[ModelType.sdcppGeneration]: sdcppConfigSchema,
|
|
33
38
|
[ModelType.audiogenGgml]: audioGenConfigSchema,
|
|
@@ -11,7 +11,7 @@ import type { Logger } from '../logging/types.js';
|
|
|
11
11
|
* The set is intentionally open-coded — adding a new kind is a one-line
|
|
12
12
|
* change and the union surfaces in editor autocomplete at every call site.
|
|
13
13
|
*/
|
|
14
|
-
export type RequestKind = 'completion' | 'batchCompletion' | 'embeddings' | 'transcribe' | 'translate' | 'diffusion' | 'world' | 'audiogen' | 'tts' | 'ocr' | 'vla' | 'finetune' | 'loadModel' | 'downloadAsset' | 'rag' | 'rpcServer' | 'rpcDiscovery';
|
|
14
|
+
export type RequestKind = 'completion' | 'batchCompletion' | 'embeddings' | 'decision' | 'transcribe' | 'translate' | 'diffusion' | 'world' | 'audiogen' | 'tts' | 'ocr' | 'vla' | 'finetune' | 'loadModel' | 'downloadAsset' | 'rag' | 'rpcServer' | 'rpcDiscovery';
|
|
15
15
|
/**
|
|
16
16
|
* Lifecycle states a request transitions through. A new context starts in
|
|
17
17
|
* `"running"`. `cancel(...)` flips it to `"cancelling"` and aborts the
|
|
@@ -21,6 +21,14 @@ let registry = null;
|
|
|
21
21
|
// passes an infinite cap and never enters the lane, so it stays ungated.
|
|
22
22
|
const LLAMACPP_COMPLETION_SLOT_GROUP = 'llamacppCompletion';
|
|
23
23
|
function installDefaultPolicies(r) {
|
|
24
|
+
// Laya owns one active job per model. Wait in the registry so cancelling a
|
|
25
|
+
// queued request never reaches the addon or interrupts the active job.
|
|
26
|
+
r.policy({
|
|
27
|
+
kind: 'decision',
|
|
28
|
+
maxConcurrentPerModel: 1,
|
|
29
|
+
onOverflow: 'queue',
|
|
30
|
+
maxQueueDepthPerModel: 64
|
|
31
|
+
});
|
|
24
32
|
// Cap is the model's `parallel`, passed per request by the handlers; the value
|
|
25
33
|
// here is only the fallback when a caller supplies none.
|
|
26
34
|
r.policy({
|
package/dist/runtime/state.js
CHANGED
|
@@ -17,6 +17,7 @@ const configRegistry = {
|
|
|
17
17
|
httpDownloadConcurrency: undefined,
|
|
18
18
|
requireHttpChecksum: undefined,
|
|
19
19
|
requireSecureTransport: undefined,
|
|
20
|
+
modelFitPolicy: undefined,
|
|
20
21
|
registryDownloadMaxRetries: undefined,
|
|
21
22
|
registryStreamTimeoutMs: undefined,
|
|
22
23
|
fitStubBudgetMs: undefined,
|
|
@@ -80,6 +81,10 @@ export function setConfig(config) {
|
|
|
80
81
|
configRegistry.requireSecureTransport = config.requireSecureTransport;
|
|
81
82
|
logger.info(`Require secure transport: ${config.requireSecureTransport}`);
|
|
82
83
|
}
|
|
84
|
+
if (config.modelFitPolicy !== undefined && config.modelFitPolicy !== null) {
|
|
85
|
+
configRegistry.modelFitPolicy = config.modelFitPolicy;
|
|
86
|
+
logger.info(`Model fit policy: ${config.modelFitPolicy}`);
|
|
87
|
+
}
|
|
83
88
|
if (config.registryDownloadMaxRetries !== undefined &&
|
|
84
89
|
config.registryDownloadMaxRetries !== null) {
|
|
85
90
|
configRegistry.registryDownloadMaxRetries = config.registryDownloadMaxRetries;
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
export declare const transcriptionFitWorkloadSchema: z.ZodObject<{
|
|
3
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
4
|
+
hotwords: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
5
|
+
maxNewTokens: z.ZodOptional<z.ZodNumber>;
|
|
6
|
+
audioSeconds: z.ZodNumber;
|
|
7
|
+
}, z.core.$strict>;
|
|
8
|
+
export type TranscriptionFitWorkload = z.infer<typeof transcriptionFitWorkloadSchema>;
|
|
2
9
|
/**
|
|
3
10
|
* Advisory outcome of a pre-download fit assessment.
|
|
4
11
|
*
|
|
@@ -19,7 +26,7 @@ export type ModelFitWorkload = {
|
|
|
19
26
|
contextTokens?: number;
|
|
20
27
|
} | {
|
|
21
28
|
kind: 'audio';
|
|
22
|
-
windowMs
|
|
29
|
+
windowMs?: number;
|
|
23
30
|
streaming: boolean;
|
|
24
31
|
};
|
|
25
32
|
/**
|
|
@@ -59,6 +66,16 @@ export declare const modelFitExecutionSchema: z.ZodEnum<{
|
|
|
59
66
|
sequential: "sequential";
|
|
60
67
|
concurrent: "concurrent";
|
|
61
68
|
}>;
|
|
69
|
+
/** Fit-only workload used to estimate MOSS text-to-speech memory requirements. */
|
|
70
|
+
export declare const ttsFitWorkloadSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
71
|
+
promptRows: z.ZodNumber;
|
|
72
|
+
referenceSamples: z.ZodNumber;
|
|
73
|
+
streaming: z.ZodBoolean;
|
|
74
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
75
|
+
prompt: z.ZodString;
|
|
76
|
+
seconds: z.ZodNumber;
|
|
77
|
+
}, z.core.$strict>]>;
|
|
78
|
+
export type TtsFitWorkload = z.infer<typeof ttsFitWorkloadSchema>;
|
|
62
79
|
/**
|
|
63
80
|
* A load to assess, in `loadModel`'s own parameters. The engine's plugin
|
|
64
81
|
* resolves it, so the config and the artifact keys the fitter reads are the
|
|
@@ -68,6 +85,12 @@ export declare const modelFitExecutionSchema: z.ZodEnum<{
|
|
|
68
85
|
* artifact, or to the file itself where it is already on disk.
|
|
69
86
|
*/
|
|
70
87
|
export declare const modelFitCandidateSchema: z.ZodObject<{
|
|
88
|
+
transcriptionWorkload: z.ZodOptional<z.ZodObject<{
|
|
89
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
90
|
+
hotwords: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
91
|
+
maxNewTokens: z.ZodOptional<z.ZodNumber>;
|
|
92
|
+
audioSeconds: z.ZodNumber;
|
|
93
|
+
}, z.core.$strict>>;
|
|
71
94
|
modelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
72
95
|
src: z.ZodString;
|
|
73
96
|
name: z.ZodOptional<z.ZodString>;
|
|
@@ -90,10 +113,12 @@ export declare const modelFitCandidateSchema: z.ZodObject<{
|
|
|
90
113
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
91
114
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
92
115
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
116
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
93
117
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
94
118
|
"onnx-tts": "onnx-tts";
|
|
95
119
|
"tts-ggml": "tts-ggml";
|
|
96
120
|
"parakeet-transcription": "parakeet-transcription";
|
|
121
|
+
"moss-transcribe": "moss-transcribe";
|
|
97
122
|
"ggml-ocr": "ggml-ocr";
|
|
98
123
|
"sdcpp-generation": "sdcpp-generation";
|
|
99
124
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -118,10 +143,12 @@ export declare const modelFitCandidateSchema: z.ZodObject<{
|
|
|
118
143
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
119
144
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
120
145
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
146
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
121
147
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
122
148
|
"onnx-tts": "onnx-tts";
|
|
123
149
|
"tts-ggml": "tts-ggml";
|
|
124
150
|
"parakeet-transcription": "parakeet-transcription";
|
|
151
|
+
"moss-transcribe": "moss-transcribe";
|
|
125
152
|
"ggml-ocr": "ggml-ocr";
|
|
126
153
|
"sdcpp-generation": "sdcpp-generation";
|
|
127
154
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -134,10 +161,24 @@ export declare const modelFitCandidateSchema: z.ZodObject<{
|
|
|
134
161
|
parakeet: "parakeet";
|
|
135
162
|
classification: "classification";
|
|
136
163
|
}>>;
|
|
164
|
+
ttsWorkload: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
165
|
+
promptRows: z.ZodNumber;
|
|
166
|
+
referenceSamples: z.ZodNumber;
|
|
167
|
+
streaming: z.ZodBoolean;
|
|
168
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
169
|
+
prompt: z.ZodString;
|
|
170
|
+
seconds: z.ZodNumber;
|
|
171
|
+
}, z.core.$strict>]>>;
|
|
137
172
|
modelConfig: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
138
173
|
}, z.core.$strip>;
|
|
139
174
|
export declare const assessModelFitInputSchema: z.ZodObject<{
|
|
140
175
|
models: z.ZodArray<z.ZodObject<{
|
|
176
|
+
transcriptionWorkload: z.ZodOptional<z.ZodObject<{
|
|
177
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
178
|
+
hotwords: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
179
|
+
maxNewTokens: z.ZodOptional<z.ZodNumber>;
|
|
180
|
+
audioSeconds: z.ZodNumber;
|
|
181
|
+
}, z.core.$strict>>;
|
|
141
182
|
modelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
142
183
|
src: z.ZodString;
|
|
143
184
|
name: z.ZodOptional<z.ZodString>;
|
|
@@ -160,10 +201,12 @@ export declare const assessModelFitInputSchema: z.ZodObject<{
|
|
|
160
201
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
161
202
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
162
203
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
204
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
163
205
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
164
206
|
"onnx-tts": "onnx-tts";
|
|
165
207
|
"tts-ggml": "tts-ggml";
|
|
166
208
|
"parakeet-transcription": "parakeet-transcription";
|
|
209
|
+
"moss-transcribe": "moss-transcribe";
|
|
167
210
|
"ggml-ocr": "ggml-ocr";
|
|
168
211
|
"sdcpp-generation": "sdcpp-generation";
|
|
169
212
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -188,10 +231,12 @@ export declare const assessModelFitInputSchema: z.ZodObject<{
|
|
|
188
231
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
189
232
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
190
233
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
234
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
191
235
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
192
236
|
"onnx-tts": "onnx-tts";
|
|
193
237
|
"tts-ggml": "tts-ggml";
|
|
194
238
|
"parakeet-transcription": "parakeet-transcription";
|
|
239
|
+
"moss-transcribe": "moss-transcribe";
|
|
195
240
|
"ggml-ocr": "ggml-ocr";
|
|
196
241
|
"sdcpp-generation": "sdcpp-generation";
|
|
197
242
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -204,6 +249,14 @@ export declare const assessModelFitInputSchema: z.ZodObject<{
|
|
|
204
249
|
parakeet: "parakeet";
|
|
205
250
|
classification: "classification";
|
|
206
251
|
}>>;
|
|
252
|
+
ttsWorkload: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
253
|
+
promptRows: z.ZodNumber;
|
|
254
|
+
referenceSamples: z.ZodNumber;
|
|
255
|
+
streaming: z.ZodBoolean;
|
|
256
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
257
|
+
prompt: z.ZodString;
|
|
258
|
+
seconds: z.ZodNumber;
|
|
259
|
+
}, z.core.$strict>]>>;
|
|
207
260
|
modelConfig: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
208
261
|
}, z.core.$strip>>;
|
|
209
262
|
execution: z.ZodDefault<z.ZodEnum<{
|
|
@@ -230,18 +283,14 @@ export declare const modelFitBasisSchema: z.ZodEnum<{
|
|
|
230
283
|
/**
|
|
231
284
|
* What kind of evidence a verdict rests on.
|
|
232
285
|
*
|
|
233
|
-
* - `
|
|
234
|
-
*
|
|
286
|
+
* - `native-fit`: the engine's own fitter, run against the model or the
|
|
287
|
+
* registry's weightless description of it. It is the answer the loader would
|
|
288
|
+
* give on this machine, and the only evidence that can confirm a fit.
|
|
235
289
|
* - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
|
|
236
290
|
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
237
291
|
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
238
|
-
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
239
|
-
* - `native-fit`: the engine's own fitter, run against the registry's weightless
|
|
240
|
-
* description of the artifact. It is the answer the loader would give on this
|
|
241
|
-
* machine, not a model of it, so it outranks `calibration` where both exist.
|
|
242
292
|
*/
|
|
243
293
|
export declare const modelFitEvidenceSchema: z.ZodEnum<{
|
|
244
|
-
calibration: "calibration";
|
|
245
294
|
"computed-only": "computed-only";
|
|
246
295
|
"native-fit": "native-fit";
|
|
247
296
|
}>;
|
|
@@ -260,14 +309,9 @@ export declare const modelFitModelResultSchema: z.ZodObject<{
|
|
|
260
309
|
"likely-too-large": "likely-too-large";
|
|
261
310
|
}>;
|
|
262
311
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
263
|
-
calibration: "calibration";
|
|
264
312
|
"computed-only": "computed-only";
|
|
265
313
|
"native-fit": "native-fit";
|
|
266
314
|
}>>;
|
|
267
|
-
estimate: z.ZodOptional<z.ZodObject<{
|
|
268
|
-
lowerBoundBytes: z.ZodNumber;
|
|
269
|
-
upperBoundBytes: z.ZodNumber;
|
|
270
|
-
}, z.core.$strip>>;
|
|
271
315
|
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
272
316
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
273
317
|
device: z.ZodOptional<z.ZodString>;
|
|
@@ -290,7 +334,6 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
290
334
|
concurrent: "concurrent";
|
|
291
335
|
}>;
|
|
292
336
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
293
|
-
calibration: "calibration";
|
|
294
337
|
"computed-only": "computed-only";
|
|
295
338
|
"native-fit": "native-fit";
|
|
296
339
|
}>>;
|
|
@@ -301,10 +344,6 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
301
344
|
reservedBytes: z.ZodNumber;
|
|
302
345
|
availableAfterReserveBytes: z.ZodNumber;
|
|
303
346
|
}, z.core.$strip>>;
|
|
304
|
-
estimate: z.ZodOptional<z.ZodObject<{
|
|
305
|
-
lowerBoundBytes: z.ZodNumber;
|
|
306
|
-
upperBoundBytes: z.ZodNumber;
|
|
307
|
-
}, z.core.$strip>>;
|
|
308
347
|
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
309
348
|
models: z.ZodArray<z.ZodObject<{
|
|
310
349
|
name: z.ZodString;
|
|
@@ -314,14 +353,9 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
314
353
|
"likely-too-large": "likely-too-large";
|
|
315
354
|
}>;
|
|
316
355
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
317
|
-
calibration: "calibration";
|
|
318
356
|
"computed-only": "computed-only";
|
|
319
357
|
"native-fit": "native-fit";
|
|
320
358
|
}>>;
|
|
321
|
-
estimate: z.ZodOptional<z.ZodObject<{
|
|
322
|
-
lowerBoundBytes: z.ZodNumber;
|
|
323
|
-
upperBoundBytes: z.ZodNumber;
|
|
324
|
-
}, z.core.$strip>>;
|
|
325
359
|
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
326
360
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
327
361
|
device: z.ZodOptional<z.ZodString>;
|
|
@@ -331,19 +365,18 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
331
365
|
assumptions: z.ZodArray<z.ZodString>;
|
|
332
366
|
}, z.core.$strip>;
|
|
333
367
|
/**
|
|
334
|
-
* The other fit question
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
338
|
-
*
|
|
368
|
+
* The other fit question: `assessModelFit` above answers "should I download
|
|
369
|
+
* this?" about a set of models against a memory budget, while the native probe
|
|
370
|
+
* answers "will the load I am about to run fit?" by handing one model file and
|
|
371
|
+
* the resolved settings to the fitter of the engine that would run it. Two
|
|
372
|
+
* scopes, so two verdict vocabularies:
|
|
339
373
|
*
|
|
340
|
-
* - `likely-fits` / `likely-too-large` hedge because
|
|
374
|
+
* - `likely-fits` / `likely-too-large` hedge because the set is judged against a
|
|
375
|
+
* sampled budget that the rest of the machine moves.
|
|
341
376
|
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
342
377
|
* file against this device.
|
|
343
378
|
*
|
|
344
|
-
*
|
|
345
|
-
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
346
|
-
* answer available before the bytes are on disk. Neither denies a load today.
|
|
379
|
+
* Neither denies a load.
|
|
347
380
|
*/
|
|
348
381
|
export declare const nativeProbeVerdictSchema: z.ZodEnum<{
|
|
349
382
|
unknown: "unknown";
|
|
@@ -364,7 +397,7 @@ export declare const nativeProbePlanSchema: z.ZodObject<{
|
|
|
364
397
|
* the engine separates them, and are absent together where it reports only a
|
|
365
398
|
* total. They are summed across devices, as `deviceBytes` is, so a load spread
|
|
366
399
|
* over more than one device is described by its totals and `deviceName` names
|
|
367
|
-
* the first.
|
|
400
|
+
* the first non-CPU row.
|
|
368
401
|
*
|
|
369
402
|
* `deviceFreeBytes` is the backend's own gauge. A device sharing the host pool,
|
|
370
403
|
* as Apple silicon and Adreno/Mali do, reports what it could address rather
|
|
@@ -379,6 +412,14 @@ export declare const nativeProbeProjectionSchema: z.ZodObject<{
|
|
|
379
412
|
computeBytes: z.ZodOptional<z.ZodNumber>;
|
|
380
413
|
deviceFreeBytes: z.ZodOptional<z.ZodNumber>;
|
|
381
414
|
deviceTotalBytes: z.ZodOptional<z.ZodNumber>;
|
|
415
|
+
devices: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
416
|
+
name: z.ZodString;
|
|
417
|
+
totalBytes: z.ZodNumber;
|
|
418
|
+
freeBytes: z.ZodNumber;
|
|
419
|
+
weightsBytes: z.ZodNumber;
|
|
420
|
+
contextBytes: z.ZodNumber;
|
|
421
|
+
computeBytes: z.ZodNumber;
|
|
422
|
+
}, z.core.$strip>>>;
|
|
382
423
|
report: z.ZodOptional<z.ZodString>;
|
|
383
424
|
}, z.core.$strip>;
|
|
384
425
|
export declare const nativeProbeFitSchema: z.ZodObject<{
|
|
@@ -406,11 +447,25 @@ export declare const nativeProbeFitSchema: z.ZodObject<{
|
|
|
406
447
|
computeBytes: z.ZodOptional<z.ZodNumber>;
|
|
407
448
|
deviceFreeBytes: z.ZodOptional<z.ZodNumber>;
|
|
408
449
|
deviceTotalBytes: z.ZodOptional<z.ZodNumber>;
|
|
450
|
+
devices: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
451
|
+
name: z.ZodString;
|
|
452
|
+
totalBytes: z.ZodNumber;
|
|
453
|
+
freeBytes: z.ZodNumber;
|
|
454
|
+
weightsBytes: z.ZodNumber;
|
|
455
|
+
contextBytes: z.ZodNumber;
|
|
456
|
+
computeBytes: z.ZodNumber;
|
|
457
|
+
}, z.core.$strip>>>;
|
|
409
458
|
report: z.ZodOptional<z.ZodString>;
|
|
410
459
|
}, z.core.$strip>>;
|
|
411
460
|
}, z.core.$strip>;
|
|
412
461
|
export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
413
462
|
models: z.ZodArray<z.ZodObject<{
|
|
463
|
+
transcriptionWorkload: z.ZodOptional<z.ZodObject<{
|
|
464
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
465
|
+
hotwords: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
466
|
+
maxNewTokens: z.ZodOptional<z.ZodNumber>;
|
|
467
|
+
audioSeconds: z.ZodNumber;
|
|
468
|
+
}, z.core.$strict>>;
|
|
414
469
|
modelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
415
470
|
src: z.ZodString;
|
|
416
471
|
name: z.ZodOptional<z.ZodString>;
|
|
@@ -433,10 +488,12 @@ export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
|
433
488
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
434
489
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
435
490
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
491
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
436
492
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
437
493
|
"onnx-tts": "onnx-tts";
|
|
438
494
|
"tts-ggml": "tts-ggml";
|
|
439
495
|
"parakeet-transcription": "parakeet-transcription";
|
|
496
|
+
"moss-transcribe": "moss-transcribe";
|
|
440
497
|
"ggml-ocr": "ggml-ocr";
|
|
441
498
|
"sdcpp-generation": "sdcpp-generation";
|
|
442
499
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -461,10 +518,12 @@ export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
|
461
518
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
462
519
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
463
520
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
521
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
464
522
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
465
523
|
"onnx-tts": "onnx-tts";
|
|
466
524
|
"tts-ggml": "tts-ggml";
|
|
467
525
|
"parakeet-transcription": "parakeet-transcription";
|
|
526
|
+
"moss-transcribe": "moss-transcribe";
|
|
468
527
|
"ggml-ocr": "ggml-ocr";
|
|
469
528
|
"sdcpp-generation": "sdcpp-generation";
|
|
470
529
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -477,6 +536,14 @@ export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
|
477
536
|
parakeet: "parakeet";
|
|
478
537
|
classification: "classification";
|
|
479
538
|
}>>;
|
|
539
|
+
ttsWorkload: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
540
|
+
promptRows: z.ZodNumber;
|
|
541
|
+
referenceSamples: z.ZodNumber;
|
|
542
|
+
streaming: z.ZodBoolean;
|
|
543
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
544
|
+
prompt: z.ZodString;
|
|
545
|
+
seconds: z.ZodNumber;
|
|
546
|
+
}, z.core.$strict>]>>;
|
|
480
547
|
modelConfig: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
481
548
|
}, z.core.$strip>>;
|
|
482
549
|
execution: z.ZodDefault<z.ZodEnum<{
|
|
@@ -503,7 +570,6 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
503
570
|
concurrent: "concurrent";
|
|
504
571
|
}>;
|
|
505
572
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
506
|
-
calibration: "calibration";
|
|
507
573
|
"computed-only": "computed-only";
|
|
508
574
|
"native-fit": "native-fit";
|
|
509
575
|
}>>;
|
|
@@ -514,10 +580,6 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
514
580
|
reservedBytes: z.ZodNumber;
|
|
515
581
|
availableAfterReserveBytes: z.ZodNumber;
|
|
516
582
|
}, z.core.$strip>>;
|
|
517
|
-
estimate: z.ZodOptional<z.ZodObject<{
|
|
518
|
-
lowerBoundBytes: z.ZodNumber;
|
|
519
|
-
upperBoundBytes: z.ZodNumber;
|
|
520
|
-
}, z.core.$strip>>;
|
|
521
583
|
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
522
584
|
models: z.ZodArray<z.ZodObject<{
|
|
523
585
|
name: z.ZodString;
|
|
@@ -527,14 +589,9 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
527
589
|
"likely-too-large": "likely-too-large";
|
|
528
590
|
}>;
|
|
529
591
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
530
|
-
calibration: "calibration";
|
|
531
592
|
"computed-only": "computed-only";
|
|
532
593
|
"native-fit": "native-fit";
|
|
533
594
|
}>>;
|
|
534
|
-
estimate: z.ZodOptional<z.ZodObject<{
|
|
535
|
-
lowerBoundBytes: z.ZodNumber;
|
|
536
|
-
upperBoundBytes: z.ZodNumber;
|
|
537
|
-
}, z.core.$strip>>;
|
|
538
595
|
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
539
596
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
540
597
|
device: z.ZodOptional<z.ZodString>;
|