@qvac/sdk 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +131 -62
- package/README.md +31 -326
- package/dist/examples/abot-world.js +1 -1
- package/dist/examples/advisory-model-fit.d.ts +0 -58
- package/dist/examples/advisory-model-fit.js +23 -150
- package/dist/examples/asr/moss-transcribe-filesystem.d.ts +2 -0
- package/dist/examples/asr/moss-transcribe-filesystem.js +84 -0
- package/dist/examples/asr/moss-transcribe-fit.d.ts +2 -0
- package/dist/examples/asr/moss-transcribe-fit.js +40 -0
- package/dist/examples/asr/nemotron-diarization-filesystem.d.ts +2 -0
- package/dist/examples/asr/nemotron-diarization-filesystem.js +64 -0
- package/dist/examples/asr/parakeet-ctc-filesystem.js +2 -2
- package/dist/examples/asr/parakeet-indic-conformer-filesystem.js +2 -2
- package/dist/examples/asr/parakeet-microphone-record.js +1 -1
- package/dist/examples/asr/parakeet-microphone-stream.js +1 -1
- package/dist/examples/asr/parakeet-nemotron-filesystem.js +3 -3
- package/dist/examples/asr/parakeet-sortformer-streaming.js +2 -2
- package/dist/examples/asr/parakeet-sortformer.js +1 -1
- package/dist/examples/asr/parakeet-tdt-filesystem-chunked.js +2 -2
- package/dist/examples/asr/parakeet-tdt-filesystem.js +2 -2
- package/dist/examples/asr/parakeet-transcribepsy-filesystem.d.ts +2 -0
- package/dist/examples/asr/parakeet-transcribepsy-filesystem.js +76 -0
- package/dist/examples/asr/parakeet-unified-coreml.js +2 -2
- package/dist/examples/asr/parakeet-unified-filesystem.js +2 -2
- package/dist/examples/asr/whispercpp-filesystem-streaming.js +1 -1
- package/dist/examples/asr/whispercpp-filesystem.js +1 -1
- package/dist/examples/asr/whispercpp-microphone-record.js +1 -1
- package/dist/examples/asr/whispercpp-prompt.js +1 -1
- package/dist/examples/assess-model-fit.js +3 -6
- package/dist/examples/audiogen/edit-music.js +2 -2
- package/dist/examples/audiogen/generate-cover.js +2 -2
- package/dist/examples/audiogen/generate-music-minimax.js +1 -1
- package/dist/examples/audiogen/generate-music.js +1 -1
- package/dist/examples/audiogen/understand-music.js +2 -2
- package/dist/examples/batch-completion.d.ts +1 -1
- package/dist/examples/batch-completion.js +1 -1
- package/dist/examples/bci/bci-filesystem-streaming.js +2 -2
- package/dist/examples/bci/bci-filesystem.js +2 -2
- package/dist/examples/concurrent-completion-collision.js +1 -1
- package/dist/examples/config-reload.js +1 -1
- package/dist/examples/decisions.d.ts +2 -0
- package/dist/examples/decisions.js +38 -0
- package/dist/examples/diffusion-esrgan-upscale.js +1 -1
- package/dist/examples/diffusion-flux2-klein-img2img.js +1 -1
- package/dist/examples/diffusion-img2img.js +1 -1
- package/dist/examples/diffusion-img2vid-ltx.js +8 -15
- package/dist/examples/diffusion-img2vid-minimax-h3.d.ts +2 -0
- package/dist/examples/diffusion-img2vid-minimax-h3.js +72 -0
- package/dist/examples/diffusion-img2vid.js +1 -1
- package/dist/examples/diffusion-qwen-image-21.d.ts +2 -0
- package/dist/examples/diffusion-qwen-image-21.js +104 -0
- package/dist/examples/diffusion-txt2vid-ltx.js +7 -14
- package/dist/examples/finetune/llamacpp-finetune.js +9 -4
- package/dist/examples/llamacpp-filesystem.js +1 -1
- package/dist/examples/llamacpp-multi-gpu.js +2 -2
- package/dist/examples/llamacpp-multimodal.js +3 -1
- package/dist/examples/llamacpp-structured-output.js +1 -1
- package/dist/examples/mcp-websearch.d.ts +2 -2
- package/dist/examples/mcp-websearch.js +4 -4
- package/dist/examples/model-fit-ui/server.d.ts +1 -1
- package/dist/examples/model-fit-ui/server.js +1 -1
- package/dist/examples/multi-job-completion.d.ts +1 -1
- package/dist/examples/multi-job-completion.js +1 -1
- package/dist/examples/ocr-fasttext.js +1 -1
- package/dist/examples/plugins.js +1 -1
- package/dist/examples/rerank.d.ts +2 -0
- package/dist/examples/rerank.js +33 -0
- package/dist/examples/tools/llamacpp-deferred-tools.js +5 -5
- package/dist/examples/tools/llamacpp-tools-auto-dialect.js +5 -4
- package/dist/examples/tools/llamacpp-tools-gemma4.js +4 -4
- package/dist/examples/tools/llamacpp-tools-qwen35.js +5 -4
- package/dist/examples/tts/llm-to-tts-streaming.d.ts +2 -2
- package/dist/examples/tts/llm-to-tts-streaming.js +2 -2
- package/dist/examples/tts/moss-sound-effect-controls.d.ts +2 -0
- package/dist/examples/tts/moss-sound-effect-controls.js +88 -0
- package/dist/examples/tts/moss-sound-effect.d.ts +2 -0
- package/dist/examples/tts/moss-sound-effect.js +86 -0
- package/dist/examples/tts/moss-speech-audio.d.ts +2 -0
- package/dist/examples/tts/moss-speech-audio.js +159 -0
- package/dist/examples/tts/moss-speech.d.ts +2 -0
- package/dist/examples/tts/moss-speech.js +70 -0
- package/dist/examples/vla-groot.js +2 -2
- package/dist/examples/vla-pi05.js +2 -2
- package/dist/examples/vla-smolvla.js +2 -2
- package/dist/examples/voice-assistant/voice-assistant.js +1 -1
- package/dist/scripts/contract/build-contract.d.ts +3 -1
- package/dist/scripts/contract/build-contract.js +40 -2
- package/dist/scripts/contract/build-model-type-maps.d.ts +6 -1
- package/dist/scripts/contract/build-model-type-maps.js +3 -2
- package/dist/scripts/contract/constants-registry.d.ts +6 -0
- package/dist/scripts/contract/constants-registry.js +4 -1
- package/dist/scripts/contract/method-shapes.d.ts +2 -0
- package/dist/scripts/contract/method-shapes.js +2 -0
- package/dist/scripts/export-contract.js +20 -7
- package/dist/scripts/run-unit-tests.js +10 -5
- package/dist/src/client/api/assess-model-fit.d.ts +5 -5
- package/dist/src/client/api/assess-model-fit.js +6 -7
- package/dist/src/client/api/audio-edit.d.ts +14 -2
- package/dist/src/client/api/audio-edit.js +16 -4
- package/dist/src/client/api/audio-gen-result.js +2 -2
- package/dist/src/client/api/audio-gen.d.ts +10 -2
- package/dist/src/client/api/audio-gen.js +12 -4
- package/dist/src/client/api/audio-understand.d.ts +1 -1
- package/dist/src/client/api/audio-understand.js +3 -3
- package/dist/src/client/api/batch-completion.js +3 -3
- package/dist/src/client/api/bci-transcribe.js +3 -3
- package/dist/src/client/api/cancel.js +1 -1
- package/dist/src/client/api/classify.js +1 -1
- package/dist/src/client/api/completion-stream.d.ts +2 -1
- package/dist/src/client/api/completion-stream.js +6 -4
- package/dist/src/client/api/decide.d.ts +31 -0
- package/dist/src/client/api/decide.js +29 -0
- package/dist/src/client/api/delete-cache.js +1 -1
- package/dist/src/client/api/diffusion.d.ts +18 -1
- package/dist/src/client/api/diffusion.js +19 -2
- package/dist/src/client/api/download-asset.js +3 -3
- package/dist/src/client/api/embed.js +2 -2
- package/dist/src/client/api/finetune.js +3 -3
- package/dist/src/client/api/get-loaded-model-info.js +1 -1
- package/dist/src/client/api/get-model-info.js +1 -1
- package/dist/src/client/api/get-system-resources.js +1 -1
- package/dist/src/client/api/heartbeat.js +1 -1
- package/dist/src/client/api/index.d.ts +5 -3
- package/dist/src/client/api/index.js +5 -3
- package/dist/src/client/api/invoke-plugin.js +1 -1
- package/dist/src/client/api/load-model.d.ts +1 -0
- package/dist/src/client/api/load-model.js +4 -4
- package/dist/src/client/api/logging-stream.d.ts +11 -2
- package/dist/src/client/api/logging-stream.js +12 -3
- package/dist/src/client/api/ocr.js +1 -1
- package/dist/src/client/api/rag.js +2 -2
- package/dist/src/client/api/registry.d.ts +2 -1
- package/dist/src/client/api/registry.js +6 -5
- package/dist/src/client/api/rerank.d.ts +22 -0
- package/dist/src/client/api/rerank.js +39 -0
- package/dist/src/client/api/resume.js +1 -1
- package/dist/src/client/api/rpc-server.js +2 -2
- package/dist/src/client/api/state.js +1 -1
- package/dist/src/client/api/subscribe-logs.js +3 -2
- package/dist/src/client/api/suspend.js +1 -1
- package/dist/src/client/api/text-to-speech.d.ts +1 -0
- package/dist/src/client/api/text-to-speech.js +18 -25
- package/dist/src/client/api/transcribe.d.ts +1 -1
- package/dist/src/client/api/transcribe.js +7 -4
- package/dist/src/client/api/translate.js +2 -2
- package/dist/src/client/api/unload-model.js +2 -2
- package/dist/src/client/api/upscale.js +1 -1
- package/dist/src/client/api/vector-index.d.ts +1 -1
- package/dist/src/client/api/vector-index.js +1 -1
- package/dist/src/client/api/video-request.d.ts +5 -5
- package/dist/src/client/api/video.d.ts +23 -3
- package/dist/src/client/api/video.js +26 -6
- package/dist/src/client/api/world-result.js +2 -2
- package/dist/src/client/api/world.d.ts +1 -1
- package/dist/src/client/api/world.js +2 -2
- package/dist/src/client/logging-stream-registry.js +9 -6
- package/dist/src/client/plugins-factory.d.ts +1 -1
- package/dist/src/client/plugins-factory.js +1 -1
- package/dist/src/client/rpc/expo-rpc-client.js +3 -3
- package/dist/src/client/rpc/ipc-socket-path.d.ts +11 -0
- package/dist/src/client/rpc/ipc-socket-path.js +30 -0
- package/dist/src/client/rpc/node-rpc-client.js +14 -25
- package/dist/src/client/rpc/rpc-client.d.ts +3 -2
- package/dist/src/client/rpc/rpc-client.js +46 -12
- package/dist/src/client/rpc/rpc-error.js +4 -1
- package/dist/src/commands/bundle/bare-pack.d.ts +8 -0
- package/dist/src/commands/bundle/bare-pack.js +28 -5
- package/dist/src/commands/bundle/constants.js +3 -0
- package/dist/src/commands/bundle/entry-gen.js +1 -1
- package/dist/src/commands/bundle/index.d.ts +8 -7
- package/dist/src/commands/bundle/index.js +135 -52
- package/dist/src/commands/bundle/manifest.d.ts +2 -0
- package/dist/src/commands/bundle/manifest.js +3 -2
- package/dist/src/commands/bundle/plugins.js +1 -1
- package/dist/src/commands/host-prebuilds/index.d.ts +18 -10
- package/dist/src/commands/host-prebuilds/index.js +39 -20
- package/dist/src/commands/index.d.ts +13 -13
- package/dist/src/commands/index.js +8 -8
- package/dist/src/commands/verify/abi.d.ts +1 -1
- package/dist/src/commands/verify/abi.js +1 -1
- package/dist/src/commands/verify/addon-source.d.ts +12 -0
- package/dist/src/commands/verify/addon-source.js +36 -3
- package/dist/src/commands/verify/bare-kit-runtime.d.ts +1 -1
- package/dist/src/commands/verify/bare-kit-runtime.js +1 -1
- package/dist/src/commands/verify/bare-kit-runtimes.js +2 -1
- package/dist/src/commands/verify/bundle-source.d.ts +1 -1
- package/dist/src/commands/verify/bundle-source.js +3 -3
- package/dist/src/commands/verify/engines-advice.d.ts +4 -4
- package/dist/src/commands/verify/engines-advice.js +4 -4
- package/dist/src/commands/verify/index.d.ts +5 -5
- package/dist/src/commands/verify/index.js +7 -7
- package/dist/src/commands/verify/node-modules-source.d.ts +1 -1
- package/dist/src/commands/verify/node-modules-source.js +11 -2
- package/dist/src/{expo/plugins/patches/qvac-platform-addons.d.ts → commands/verify/platform-addons.d.ts} +6 -10
- package/dist/src/{expo/plugins/patches/qvac-platform-addons.js → commands/verify/platform-addons.js} +2 -45
- package/dist/src/commands/verify/prebuilds.d.ts +8 -11
- package/dist/src/commands/verify/prebuilds.js +44 -35
- package/dist/src/commands/verify/resource-collectors.js +2 -2
- package/dist/src/expo/plugins/link-addons.d.ts +18 -0
- package/dist/src/expo/plugins/link-addons.js +65 -0
- package/dist/src/expo/plugins/withAndroidNdkVersion.js +16 -10
- package/dist/src/expo/plugins/withDeviceInfo.js +19 -5
- package/dist/src/expo/plugins/withMobileBundle.d.ts +13 -23
- package/dist/src/expo/plugins/withMobileBundle.js +27 -117
- package/dist/src/index.d.ts +4 -4
- package/dist/src/index.js +3 -3
- package/dist/src/logging/addon.d.ts +1 -1
- package/dist/src/pear/pre.js +4 -0
- package/dist/src/plugin-ids.d.ts +4 -1
- package/dist/src/plugin-ids.js +6 -0
- package/dist/src/schemas/public.d.ts +2 -1
- package/dist/src/schemas/public.js +2 -1
- package/dist/src/schemas/sdk-errors-client.d.ts +3 -0
- package/dist/src/schemas/sdk-errors-client.js +16 -0
- package/dist/src/schemas/sdk-errors-server.d.ts +2 -0
- package/dist/src/schemas/sdk-errors-server.js +11 -1
- package/dist/src/server/rpc/rpc-utils.d.ts +5 -0
- package/dist/src/server/rpc/rpc-utils.js +7 -0
- package/dist/src/server/utils/qvac-paths.js +1 -1
- package/dist/src/server/utils/worker-lock.js +1 -1
- package/dist/src/utils/aggregate-events.d.ts +1 -1
- package/dist/src/utils/aggregate-events.js +2 -2
- package/dist/src/utils/errors-client.d.ts +10 -0
- package/dist/src/utils/errors-client.js +18 -0
- package/dist/src/utils/errors-server.d.ts +6 -0
- package/dist/src/utils/errors-server.js +12 -0
- package/dist/src/utils/mcp-adapter.d.ts +1 -1
- package/dist/src/utils/mcp-adapter.js +13 -9
- package/dist/src/utils/tool-helpers.d.ts +1 -1
- package/dist/src/utils/tool-helpers.js +21 -53
- package/dist/src/worker/create-server.d.ts +1 -0
- package/dist/src/worker/create-server.js +5 -0
- package/dist/src/worker/handle-request.js +4 -2
- package/dist/src/worker/handler-utils.js +1 -1
- package/dist/src/worker/index.js +7 -1
- package/dist/src/worker/lifecycle.js +10 -2
- package/dist/src/worker/plugins/llamacpp-decision.d.ts +2 -0
- package/dist/src/worker/plugins/llamacpp-decision.js +1 -0
- package/dist/src/worker/plugins/llamacpp-rerank.d.ts +2 -0
- package/dist/src/worker/plugins/llamacpp-rerank.js +1 -0
- package/dist/src/worker/plugins/moss-transcribe.d.ts +2 -0
- package/dist/src/worker/plugins/moss-transcribe.js +1 -0
- package/package.json +63 -53
- package/src/electron-forge/index.cjs +12 -71
- package/dist/scripts/link-workspace-inference.d.ts +0 -2
- package/dist/scripts/link-workspace-inference.js +0 -31
- package/src/expo/plugins/patches/android-link.mjs +0 -67
- package/src/expo/plugins/patches/ios-link.mjs +0 -67
- package/src/expo/plugins/patches/qvac-platform-addons.ts +0 -124
package/README.md
CHANGED
|
@@ -29,6 +29,8 @@ package manager can replace an older, incompatible runtime. Remove any override
|
|
|
29
29
|
or resolution that forces the worker below that requirement; installing a newer
|
|
30
30
|
global `bare` executable does not change the SDK's worker dependency.
|
|
31
31
|
|
|
32
|
+
For mobile apps that use only raw PCM streaming transcription, list the required audio plugins and set `"includeAudioDecoder": false` in `qvac.config.*` before bundling or running Expo prebuild. You can also use `qvac bundle sdk --defer bare-ffmpeg`. Both options leave `bare-ffmpeg` out of the native addon manifest. Bundle FFmpeg when the app decodes compressed audio files or asks `audiogen-ggml` for a compressed output format; PCM and WAV output need no FFmpeg.
|
|
33
|
+
|
|
32
34
|
## Quickstart
|
|
33
35
|
|
|
34
36
|
1. Create the examples workspace:
|
|
@@ -82,343 +84,46 @@ try {
|
|
|
82
84
|
node quickstart.js
|
|
83
85
|
```
|
|
84
86
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
```ts
|
|
92
|
-
import { getSystemResources } from '@qvac/sdk'
|
|
93
|
-
|
|
94
|
-
const resources = await getSystemResources({ sample: true })
|
|
95
|
-
|
|
96
|
-
if (resources.capabilities.memory.totalBytes.status === 'supported') {
|
|
97
|
-
console.log('System memory:', resources.capabilities.memory.totalBytes.value)
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
if (resources.sample?.cpu.status === 'supported') {
|
|
101
|
-
console.log('CPU utilization:', resources.sample.cpu.value)
|
|
102
|
-
}
|
|
103
|
-
```
|
|
104
|
-
|
|
105
|
-
See the [system resources support matrix](./docs/system-resources-support-matrix.md)
|
|
106
|
-
for metric-level evidence and platform limitations.
|
|
107
|
-
|
|
108
|
-
Every metric reports `supported`, `unavailable`, `unverified`, or `failed`.
|
|
109
|
-
Supported values include provenance with a source and optional scope. These
|
|
110
|
-
values are diagnostics; they do not reserve memory or guarantee that a model
|
|
111
|
-
can be loaded.
|
|
112
|
-
|
|
113
|
-
GPU capabilities expose observed driver names, versions, and graphics APIs.
|
|
114
|
-
These observations do not prove that an inference backend is compatible.
|
|
115
|
-
|
|
116
|
-
Profiled inference operation events may include `event.backend` with the
|
|
117
|
-
selected backend and device, graphics API, driver, fallback reason, and probe
|
|
118
|
-
result. Addons attach backend metadata with `attachBackendDiagnostics`; the SDK
|
|
119
|
-
validates it before recording the operation event. `gpuId`, when present,
|
|
120
|
-
identifies a GPU from the current worker resource collector and is stable only
|
|
121
|
-
for that collector's lifetime. The SDK does not infer compatibility from driver
|
|
122
|
-
inventory or log text.
|
|
123
|
-
|
|
124
|
-
### Profiler resource gauges
|
|
125
|
-
|
|
126
|
-
Resource gauges are disabled by default. Enable them explicitly to attach one
|
|
127
|
-
worker resource sample to each profiled operation:
|
|
128
|
-
|
|
129
|
-
```ts
|
|
130
|
-
import { profiler } from '@qvac/sdk'
|
|
131
|
-
|
|
132
|
-
profiler.enable({ mode: 'verbose', includeResourceGauges: true })
|
|
133
|
-
|
|
134
|
-
// Run SDK operations, then inspect recentEvents[].resources.
|
|
135
|
-
const profile = profiler.exportJSON()
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
The sample uses the same status, provenance, and scope semantics as
|
|
139
|
-
`getSystemResources({ sample: true })`. Its `sampledAt` uses the same monotonic
|
|
140
|
-
clock as the profiling event's `ts`, so the two timestamps are comparable.
|
|
141
|
-
`resources.origin` records that the sample was taken on the `local` worker.
|
|
142
|
-
Samples are delivered to `profiler.onRecord`; they
|
|
143
|
-
are retained in `exportJSON().recentEvents` only in `verbose` mode. Enabling
|
|
144
|
-
gauges in `summary` mode still incurs the sampling cost without retaining them.
|
|
145
|
-
Disabling profiling or omitting `includeResourceGauges` performs no resource
|
|
146
|
-
sampling. Enabling gauges adds one CPU query and one query per GPU to each
|
|
147
|
-
profiled operation's response path. If the worker resource collector is not
|
|
148
|
-
initialized, the event omits the resource block.
|
|
149
|
-
|
|
150
|
-
## Pre-download model fit assessment
|
|
151
|
-
|
|
152
|
-
Use `assessModelFit` to check, before downloading anything, whether models are
|
|
153
|
-
likely to fit in this device's memory. Each candidate is described as `loadModel`
|
|
154
|
-
takes it. It reads generated catalog metadata plus a fresh memory sample, and for
|
|
155
|
-
a single candidate the registry's weightless description of the artifact, so the
|
|
156
|
-
engine's own fitter can answer — no weights, no load:
|
|
157
|
-
|
|
158
|
-
```ts
|
|
159
|
-
import { assessModelFit, QWEN3_8B_INST_Q4_K_M } from '@qvac/sdk'
|
|
160
|
-
|
|
161
|
-
const result = await assessModelFit({
|
|
162
|
-
models: [{ modelSrc: QWEN3_8B_INST_Q4_K_M, modelType: 'llm', modelConfig: { ctx_size: 8192 } }],
|
|
163
|
-
execution: 'sequential',
|
|
164
|
-
policy: 'interactive-v1'
|
|
165
|
-
})
|
|
166
|
-
|
|
167
|
-
console.log(result.verdict) // 'likely-fits' | 'likely-too-large' | 'unknown'
|
|
168
|
-
```
|
|
169
|
-
|
|
170
|
-
The result is advisory: it does not block `loadModel`, reserve memory, or make a
|
|
171
|
-
performance claim. `unknown` is a real answer meaning the evidence does not
|
|
172
|
-
support a call either way — show it as "can't say", not as "no".
|
|
173
|
-
|
|
174
|
-
See [pre-download model fit assessment](./docs/assess-model-fit.md) for the
|
|
175
|
-
budget arithmetic, why estimates are ranges, the supported engine and workload
|
|
176
|
-
matrix, and the current calibration status.
|
|
177
|
-
|
|
178
|
-
## Streaming transcription statistics
|
|
179
|
-
|
|
180
|
-
Whisper and Parakeet duplex transcription sessions expose terminal engine
|
|
181
|
-
statistics after their event iterator completes:
|
|
182
|
-
|
|
183
|
-
```ts
|
|
184
|
-
const session = await transcribeStream({ modelId })
|
|
185
|
-
|
|
186
|
-
for await (const text of session) {
|
|
187
|
-
process.stdout.write(text)
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
const stats = await session.stats
|
|
191
|
-
console.log(stats?.audioDuration, stats?.realTimeFactor)
|
|
192
|
-
```
|
|
193
|
-
|
|
194
|
-
`session.stats` resolves to `undefined` when the engine does not report
|
|
195
|
-
statistics.
|
|
87
|
+
A first `loadModel()` call also starts the SDK's worker. If that startup ever
|
|
88
|
+
exceeds its budget (cold disk cache, antivirus scanning the native addon), the
|
|
89
|
+
call fails with `RPC_INIT_TIMEOUT`. Re-running usually succeeds; to give every
|
|
90
|
+
start more room, raise `rpcInitTimeoutMs` in `qvac.config.*` or set the
|
|
91
|
+
`QVAC_RPC_INIT_TIMEOUT_MS` environment variable.
|
|
196
92
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
```ts
|
|
200
|
-
const call = transcribe({ modelId, audioChunk })
|
|
201
|
-
const text = await call
|
|
202
|
-
const stats = await call.stats
|
|
203
|
-
console.log(text, stats?.encoderUsedCoreml)
|
|
204
|
-
```
|
|
93
|
+
## Documentation
|
|
205
94
|
|
|
206
|
-
|
|
207
|
-
|
|
95
|
+
The [QVAC SDK documentation](https://docs.qvac.tether.io/sdk/) covers each
|
|
96
|
+
capability with its configuration and examples. See the
|
|
97
|
+
[API reference](https://docs.qvac.tether.io/sdk/reference/api/) for the full API
|
|
98
|
+
and the [release notes](https://docs.qvac.tether.io/sdk/reference/release-notes/)
|
|
99
|
+
for what changed in each release.
|
|
208
100
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
Sortformer v2.1 also needs a GGUF with the
|
|
219
|
-
`parakeet.model_variant=sortformer-streaming-v2.1-aosc` metadata. Caching both
|
|
220
|
-
sidecars alone does not activate Core ML. See
|
|
221
|
-
[`examples/asr/parakeet-unified-coreml.ts`](examples/asr/parakeet-unified-coreml.ts)
|
|
222
|
-
for a local check of a Unified GGUF and its encoder bundle.
|
|
101
|
+
TranscribePsy (`PARAKEET_TRANSCRIBEPSY_250M_Q8_0`, eight European
|
|
102
|
+
languages) has a separate Core ML distribution instead of a sidecar:
|
|
103
|
+
`PARAKEET_TRANSCRIBEPSY_250M_COREML_Q8_0` is an encoder-less GGUF whose
|
|
104
|
+
encoder and decoder bundles the SDK stages beside it, so both stages run on
|
|
105
|
+
the Neural Engine. It loads on macOS and iOS only; elsewhere the load is
|
|
106
|
+
refused up front with the ggml constant to use instead. See
|
|
107
|
+
[`examples/asr/parakeet-transcribepsy-filesystem.ts`](examples/asr/parakeet-transcribepsy-filesystem.ts),
|
|
108
|
+
which also shows the `longFormWindowFrames` / `longFormContextFrames` knobs
|
|
109
|
+
that bound the offline encoder window on long recordings.
|
|
223
110
|
|
|
224
111
|
## Examples
|
|
225
112
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
2. Run using Bare, Node.js, or Bun as the runtime:
|
|
113
|
+
The [`examples`](./examples) directory has scripts covering every SDK capability.
|
|
114
|
+
To run one, set up the SDK from source as described in
|
|
115
|
+
[`CONTRIBUTING.md`](./CONTRIBUTING.md), then from `packages/sdk`:
|
|
230
116
|
|
|
231
117
|
```bash
|
|
232
|
-
#
|
|
233
|
-
bun run bare:example dist/examples/path/to/example.js
|
|
234
|
-
|
|
235
|
-
# With Node
|
|
118
|
+
# Compiled output
|
|
236
119
|
node dist/examples/path/to/example.js
|
|
237
120
|
|
|
238
|
-
#
|
|
239
|
-
|
|
240
|
-
```
|
|
241
|
-
|
|
242
|
-
`examples/abot-world.ts` has a companion guide covering the hardware
|
|
243
|
-
requirements, scene-pack lifecycle, cancellation semantics and concurrency rules
|
|
244
|
-
of an interactive world session: see
|
|
245
|
-
[ABot-World interactive world sessions](./docs/abot-world.md).
|
|
246
|
-
|
|
247
|
-
## Build
|
|
248
|
-
|
|
249
|
-
Use the [Bun](https://bun.sh/) package manager:
|
|
250
|
-
|
|
251
|
-
```bash
|
|
252
|
-
bun i
|
|
253
|
-
```
|
|
254
|
-
|
|
255
|
-
`@qvac/inference` resolves to its published release by default. To build and test against the in-repo engine at the same commit, link it first:
|
|
256
|
-
|
|
257
|
-
```bash
|
|
258
|
-
bun run sdk-source:workspace
|
|
121
|
+
# Straight from source, with the tsx dev dependency
|
|
122
|
+
npx tsx examples/path/to/example.ts
|
|
259
123
|
```
|
|
260
124
|
|
|
261
|
-
```bash
|
|
262
|
-
bun run build # or `watch` for hotreload
|
|
263
|
-
```
|
|
264
|
-
|
|
265
|
-
```bash
|
|
266
|
-
bun run build:pack
|
|
267
|
-
```
|
|
268
|
-
|
|
269
|
-
This outputs a tarball under `dist/sdk-{version}.tgz` that you can install in your project, e.g.:
|
|
270
|
-
|
|
271
|
-
```bash
|
|
272
|
-
npm i path/to/sdk-0.3.0.tgz
|
|
273
|
-
```
|
|
274
|
-
|
|
275
|
-
## Testing
|
|
276
|
-
|
|
277
|
-
The SDK test suite is organized into two buckets by runtime:
|
|
278
|
-
|
|
279
|
-
| Bucket | Runtime | Location | Command |
|
|
280
|
-
| ----------------- | ---------- | -------- | -------------------------------------- |
|
|
281
|
-
| Unit | Bun / Node | `test/` | `bun run test:unit` |
|
|
282
|
-
| Client (consumer) | Node / RN | `e2e/` | See [`e2e/README.md`](./e2e/README.md) |
|
|
283
|
-
|
|
284
|
-
See [`TESTING.md`](./TESTING.md) for the full decision tree on where new tests should land.
|
|
285
|
-
|
|
286
125
|
## Contributing
|
|
287
126
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
#### Format
|
|
293
|
-
|
|
294
|
-
**Commit messages:**
|
|
295
|
-
|
|
296
|
-
```
|
|
297
|
-
prefix[tags]?: subject
|
|
298
|
-
```
|
|
299
|
-
|
|
300
|
-
**PR titles:**
|
|
301
|
-
|
|
302
|
-
```
|
|
303
|
-
TICKET prefix[tags]: subject
|
|
304
|
-
```
|
|
305
|
-
|
|
306
|
-
#### Allowed Prefixes
|
|
307
|
-
|
|
308
|
-
- `feat` - New features or capabilities
|
|
309
|
-
- `fix` - Bug fixes
|
|
310
|
-
- `doc` - Documentation changes
|
|
311
|
-
- `test` - Test additions or modifications
|
|
312
|
-
- `mod` - Model-related changes
|
|
313
|
-
- `chore` - Maintenance tasks
|
|
314
|
-
- `infra` - CI/CD, tooling, infrastructure
|
|
315
|
-
|
|
316
|
-
#### Allowed Tags
|
|
317
|
-
|
|
318
|
-
Tags are optional:
|
|
319
|
-
|
|
320
|
-
- `[api]` - API changes (non-breaking)
|
|
321
|
-
- `[bc]` - Breaking changes (including breaking API changes)
|
|
322
|
-
|
|
323
|
-
#### Examples
|
|
324
|
-
|
|
325
|
-
**Valid commit messages:**
|
|
326
|
-
|
|
327
|
-
```bash
|
|
328
|
-
feat: add RAG support for LanceDB
|
|
329
|
-
fix[api]: fix completion stream error handling
|
|
330
|
-
doc: update installation instructions
|
|
331
|
-
feat[bc]: redesign loadModel signature
|
|
332
|
-
chore: update dependencies
|
|
333
|
-
```
|
|
334
|
-
|
|
335
|
-
**Valid PR titles:**
|
|
336
|
-
|
|
337
|
-
```bash
|
|
338
|
-
QVAC-123 feat: add RAG support for LanceDB
|
|
339
|
-
QVAC-456 fix[api]: fix completion stream error handling
|
|
340
|
-
QVAC-789 doc: update installation instructions
|
|
341
|
-
QVAC-101 feat[bc]: redesign loadModel signature
|
|
342
|
-
```
|
|
343
|
-
|
|
344
|
-
#### Code Examples Requirements
|
|
345
|
-
|
|
346
|
-
When creating PRs with specific tags, you must include code examples in the PR description:
|
|
347
|
-
|
|
348
|
-
**`[bc]` tag requirements:**
|
|
349
|
-
|
|
350
|
-
Must include BEFORE/AFTER code examples showing the migration path:
|
|
351
|
-
|
|
352
|
-
````markdown
|
|
353
|
-
## BC Changes
|
|
354
|
-
|
|
355
|
-
**BEFORE:**
|
|
356
|
-
|
|
357
|
-
```typescript
|
|
358
|
-
const model = await loadModel('model-path')
|
|
359
|
-
```
|
|
360
|
-
|
|
361
|
-
**AFTER:**
|
|
362
|
-
|
|
363
|
-
```typescript
|
|
364
|
-
const modelId = await loadModel('model-path', { modelType: 'llm' })
|
|
365
|
-
```
|
|
366
|
-
````
|
|
367
|
-
|
|
368
|
-
Or using inline comments:
|
|
369
|
-
|
|
370
|
-
````markdown
|
|
371
|
-
```typescript
|
|
372
|
-
// old
|
|
373
|
-
const model = await loadModel('model-path')
|
|
374
|
-
|
|
375
|
-
// new
|
|
376
|
-
const modelId = await loadModel('model-path', { modelType: 'llm' })
|
|
377
|
-
```
|
|
378
|
-
````
|
|
379
|
-
|
|
380
|
-
**`[api]` tag requirements (non-breaking):**
|
|
381
|
-
|
|
382
|
-
Must include at least one fenced code block showing the new API usage:
|
|
383
|
-
|
|
384
|
-
````markdown
|
|
385
|
-
## New API
|
|
386
|
-
|
|
387
|
-
```typescript
|
|
388
|
-
// New completion API with streaming support
|
|
389
|
-
for await (const token of completion({
|
|
390
|
-
modelId,
|
|
391
|
-
history: [{ role: 'user', content: 'Hello!' }]
|
|
392
|
-
}).tokenStream) {
|
|
393
|
-
process.stdout.write(token)
|
|
394
|
-
}
|
|
395
|
-
```
|
|
396
|
-
````
|
|
397
|
-
|
|
398
|
-
#### Validation
|
|
399
|
-
|
|
400
|
-
- **Commit messages** are validated automatically via Husky commit-msg hook
|
|
401
|
-
- **PR titles and descriptions** are validated via GitHub Actions on PR creation/update
|
|
402
|
-
- Invalid commits or PRs will be rejected with helpful error messages
|
|
403
|
-
- **Auto-skipped commits:** The following Git-generated commits bypass validation:
|
|
404
|
-
- Merge commits (e.g., `Merge pull request #123`)
|
|
405
|
-
- Version bumps (e.g., `1.0.0`, `v1.0.0`)
|
|
406
|
-
- Revert commits (e.g., `Revert "feat: add feature"`)
|
|
407
|
-
- Squash commits (e.g., `squash! fix: bug fix`)
|
|
408
|
-
|
|
409
|
-
#### Generating Changelogs
|
|
410
|
-
|
|
411
|
-
Once your PRs are merged into `dev`, you can generate a changelog:
|
|
412
|
-
|
|
413
|
-
```bash
|
|
414
|
-
npm run changelog:generate
|
|
415
|
-
```
|
|
416
|
-
|
|
417
|
-
This will:
|
|
418
|
-
|
|
419
|
-
1. Compare versions between `dev` and `main` branches
|
|
420
|
-
2. Collect all merged PRs
|
|
421
|
-
3. Parse and classify each PR by prefix
|
|
422
|
-
4. Generate `changelog/<version>/CHANGELOG.md`
|
|
423
|
-
5. Generate `changelog/<version>/breaking.md` for BC changes (with code examples)
|
|
424
|
-
6. Generate `changelog/<version>/api.md` for API changes (with code examples)
|
|
127
|
+
See [`CONTRIBUTING.md`](./CONTRIBUTING.md) for building from source, commit and
|
|
128
|
+
PR title format, and changelog generation, and [`TESTING.md`](./TESTING.md) for
|
|
129
|
+
where new tests belong.
|
|
@@ -17,7 +17,7 @@ const width = Number(process.argv[6] || 832);
|
|
|
17
17
|
const height = Number(process.argv[7] || 480);
|
|
18
18
|
if (!firstFramePath) {
|
|
19
19
|
console.error('✖ first frame image path is required');
|
|
20
|
-
console.error('Usage:
|
|
20
|
+
console.error('Usage: node dist/examples/abot-world.js ' +
|
|
21
21
|
'<firstFrameImage> [prompt] [outputDir] [blocks] [width] [height]');
|
|
22
22
|
process.exit(1);
|
|
23
23
|
}
|
|
@@ -1,60 +1,2 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Advisory model fit check.
|
|
3
|
-
*
|
|
4
|
-
* Before a load, the SDK runs the fitter belonging to the engine that would
|
|
5
|
-
* run it and projects whether the exact configuration it is about to load will
|
|
6
|
-
* fit in device memory. Desktop runs it in a disposable Bare child; mobile,
|
|
7
|
-
* which cannot spawn one, runs it on a worker thread.
|
|
8
|
-
*
|
|
9
|
-
* The result is ADVISORY. It never blocks a load. `does-not-fit` is logged and
|
|
10
|
-
* the ordinary load path runs unchanged. Crashes, timeouts, malformed
|
|
11
|
-
* responses, unsupported configurations, and internal errors all resolve to
|
|
12
|
-
* "no evidence" and are equally non-blocking. No verdict changes the load;
|
|
13
|
-
* `getLoadedModelInfo` returns it as `fitProbe`.
|
|
14
|
-
*
|
|
15
|
-
* The verdict is emitted on the SDK server log stream, not to stdout, so this
|
|
16
|
-
* example subscribes to `loggingStream({ id: SDK_LOG_ID })` and reprints the
|
|
17
|
-
* `[advisory-fit:…]` lines.
|
|
18
|
-
*
|
|
19
|
-
* ---------------------------------------------------------------------------
|
|
20
|
-
* What fits on this machine (Apple M4 Pro, 24 GiB unified memory)
|
|
21
|
-
* ---------------------------------------------------------------------------
|
|
22
|
-
*
|
|
23
|
-
* Measured at the default 1024 MiB margin. The fitter budgets against what the
|
|
24
|
-
* machine can actually keep resident
|
|
25
|
-
* (total − wired − compressor: 17.4 GiB on this machine at idle), not the raw
|
|
26
|
-
* RAM figure.
|
|
27
|
-
*
|
|
28
|
-
* PROJECTED TO FIT — all layers on GPU
|
|
29
|
-
* Qwen3.5 0.8B Q4_K_M 0.5 GiB @ 4k ctx
|
|
30
|
-
* gpt-oss-20B Q4_K_M 10.8 GiB @ 32k ctx
|
|
31
|
-
* gte-large fp16 0.6 GiB (embedding, context pinned to 512)
|
|
32
|
-
*
|
|
33
|
-
* PROJECTED NOT TO FIT — with `gpu_layers: 99` pinned
|
|
34
|
-
* gpt-oss-20B Q4_K_M 10.8 GiB @ 128k ctx with an f32 KV cache ← below
|
|
35
|
-
*
|
|
36
|
-
* These rows were measured with `gpu_layers` pinned at 99. Loads no longer pin
|
|
37
|
-
* the layer count, and the `does-not-fit` row has not been re-measured
|
|
38
|
-
* unpinned.
|
|
39
|
-
*
|
|
40
|
-
* gpt-oss-20B at 128k context FITS with the default KV cache and DOES NOT FIT
|
|
41
|
-
* once `cache-type-k`/`cache-type-v` are set to `f32`. Same model, same
|
|
42
|
-
* context, same machine — the verdict tracks the configuration, not the file
|
|
43
|
-
* size.
|
|
44
|
-
*
|
|
45
|
-
* Two boundaries worth understanding when reading verdicts:
|
|
46
|
-
*
|
|
47
|
-
* 1. The verdict answers for a PLACEMENT, not a model. With `gpu_layers`
|
|
48
|
-
* unset, the fitter may move layers to the CPU side; setting it pins the
|
|
49
|
-
* layer count, and `does-not-fit` then means "not at this placement".
|
|
50
|
-
* 2. A `does-not-fit` configuration can still RUN on macOS when the OS
|
|
51
|
-
* compresses and pages hard enough — and a `fits` configuration right at
|
|
52
|
-
* the boundary can still fail at first decode under memory pressure.
|
|
53
|
-
* Prediction cannot separate those cases from a snapshot; the addon-side
|
|
54
|
-
* probe decode (QVAC-24114) is the runtime check that catches the
|
|
55
|
-
* remainder. The verdict here is the honest working-set budget, and the
|
|
56
|
-
* measured failure modes punish over-committing, so treat `does-not-fit`
|
|
57
|
-
* as "expect degradation or decode failure", not "the load will error".
|
|
58
|
-
*/
|
|
59
1
|
export {};
|
|
60
2
|
//# sourceMappingURL=advisory-model-fit.d.ts.map
|
|
@@ -1,156 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
* "no evidence" and are equally non-blocking. No verdict changes the load;
|
|
13
|
-
* `getLoadedModelInfo` returns it as `fitProbe`.
|
|
14
|
-
*
|
|
15
|
-
* The verdict is emitted on the SDK server log stream, not to stdout, so this
|
|
16
|
-
* example subscribes to `loggingStream({ id: SDK_LOG_ID })` and reprints the
|
|
17
|
-
* `[advisory-fit:…]` lines.
|
|
18
|
-
*
|
|
19
|
-
* ---------------------------------------------------------------------------
|
|
20
|
-
* What fits on this machine (Apple M4 Pro, 24 GiB unified memory)
|
|
21
|
-
* ---------------------------------------------------------------------------
|
|
22
|
-
*
|
|
23
|
-
* Measured at the default 1024 MiB margin. The fitter budgets against what the
|
|
24
|
-
* machine can actually keep resident
|
|
25
|
-
* (total − wired − compressor: 17.4 GiB on this machine at idle), not the raw
|
|
26
|
-
* RAM figure.
|
|
27
|
-
*
|
|
28
|
-
* PROJECTED TO FIT — all layers on GPU
|
|
29
|
-
* Qwen3.5 0.8B Q4_K_M 0.5 GiB @ 4k ctx
|
|
30
|
-
* gpt-oss-20B Q4_K_M 10.8 GiB @ 32k ctx
|
|
31
|
-
* gte-large fp16 0.6 GiB (embedding, context pinned to 512)
|
|
32
|
-
*
|
|
33
|
-
* PROJECTED NOT TO FIT — with `gpu_layers: 99` pinned
|
|
34
|
-
* gpt-oss-20B Q4_K_M 10.8 GiB @ 128k ctx with an f32 KV cache ← below
|
|
35
|
-
*
|
|
36
|
-
* These rows were measured with `gpu_layers` pinned at 99. Loads no longer pin
|
|
37
|
-
* the layer count, and the `does-not-fit` row has not been re-measured
|
|
38
|
-
* unpinned.
|
|
39
|
-
*
|
|
40
|
-
* gpt-oss-20B at 128k context FITS with the default KV cache and DOES NOT FIT
|
|
41
|
-
* once `cache-type-k`/`cache-type-v` are set to `f32`. Same model, same
|
|
42
|
-
* context, same machine — the verdict tracks the configuration, not the file
|
|
43
|
-
* size.
|
|
44
|
-
*
|
|
45
|
-
* Two boundaries worth understanding when reading verdicts:
|
|
46
|
-
*
|
|
47
|
-
* 1. The verdict answers for a PLACEMENT, not a model. With `gpu_layers`
|
|
48
|
-
* unset, the fitter may move layers to the CPU side; setting it pins the
|
|
49
|
-
* layer count, and `does-not-fit` then means "not at this placement".
|
|
50
|
-
* 2. A `does-not-fit` configuration can still RUN on macOS when the OS
|
|
51
|
-
* compresses and pages hard enough — and a `fits` configuration right at
|
|
52
|
-
* the boundary can still fail at first decode under memory pressure.
|
|
53
|
-
* Prediction cannot separate those cases from a snapshot; the addon-side
|
|
54
|
-
* probe decode (QVAC-24114) is the runtime check that catches the
|
|
55
|
-
* remainder. The verdict here is the honest working-set budget, and the
|
|
56
|
-
* measured failure modes punish over-committing, so treat `does-not-fit`
|
|
57
|
-
* as "expect degradation or decode failure", not "the load will error".
|
|
58
|
-
*/
|
|
59
|
-
import { completion, loadModel, unloadModel, loggingStream, SDK_LOG_ID, QWEN3_5_0_8B_MULTIMODAL_Q4_K_M, GPT_OSS_20B_INST_Q4_K_M } from '@qvac/sdk';
|
|
60
|
-
// The oversized load below is expected to be reported as `does-not-fit` and
|
|
61
|
-
// then attempted anyway, because the check is advisory. It really does try to
|
|
62
|
-
// allocate ~11 GiB, so it stays opt-in.
|
|
63
|
-
const ATTEMPT_OVERSIZED = process.env['QVAC_FIT_DEMO_ATTEMPT_OVERSIZED'] === '1';
|
|
64
|
-
// Reprint the worker's advisory verdicts. They arrive on the SDK server log
|
|
65
|
-
// stream; everything else on that stream is filtered out to keep this readable.
|
|
66
|
-
//
|
|
67
|
-
// Called once per phase rather than once for the process: a subscription
|
|
68
|
-
// currently stops delivering after any `unloadModel`, so a single one would go
|
|
69
|
-
// silent before the second verdict. Resubscribing after the unload works.
|
|
70
|
-
function watchVerdicts() {
|
|
71
|
-
void (async () => {
|
|
72
|
-
for await (const log of loggingStream({ id: SDK_LOG_ID })) {
|
|
73
|
-
if (log.message.includes('[advisory-fit:')) {
|
|
74
|
-
console.log(`▸ [${log.level.toUpperCase()}] ${log.message}`);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
})().catch(() => {
|
|
78
|
-
// Stream terminated — normal on shutdown.
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
watchVerdicts();
|
|
1
|
+
// `modelFitPolicy` decides what the engine fitter's verdict does to a load:
|
|
2
|
+
// `log` (the default) reports it and loads anyway, `refuse` rejects a load the
|
|
3
|
+
// fitter projects will not fit, `off` skips the check.
|
|
4
|
+
import { loadModel, loggingStream, SDK_LOG_ID, GPT_OSS_20B_INST_Q4_K_M } from '@qvac/sdk';
|
|
5
|
+
// Verdicts go to the SDK server log stream, not stdout.
|
|
6
|
+
void (async () => {
|
|
7
|
+
for await (const log of loggingStream({ id: SDK_LOG_ID })) {
|
|
8
|
+
if (log.message.includes('[advisory-fit:'))
|
|
9
|
+
console.log(`▸ ${log.message}`);
|
|
10
|
+
}
|
|
11
|
+
})().catch(() => { });
|
|
82
12
|
try {
|
|
83
|
-
//
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
13
|
+
// gpt-oss-20B fits here at the default KV cache type and does not at f32.
|
|
14
|
+
await loadModel({
|
|
15
|
+
modelSrc: GPT_OSS_20B_INST_Q4_K_M,
|
|
16
|
+
modelConfig: {
|
|
17
|
+
ctx_size: 131072,
|
|
18
|
+
'cache-type-k': 'f32',
|
|
19
|
+
'cache-type-v': 'f32'
|
|
20
|
+
},
|
|
21
|
+
modelFitPolicy: 'refuse'
|
|
89
22
|
});
|
|
90
|
-
console.log(
|
|
91
|
-
|
|
92
|
-
modelId: smallModelId,
|
|
93
|
-
history: [{ role: 'user', content: 'Say hello in five words.' }],
|
|
94
|
-
stream: false,
|
|
95
|
-
generationParams: { predict: 48 }
|
|
96
|
-
});
|
|
97
|
-
const final = await result.final;
|
|
98
|
-
console.log(`▸ Completion still works normally: ${final.contentText.trim().slice(0, 120)}\n`);
|
|
99
|
-
// Unloaded before the next phase, so the second verdict is measured on an
|
|
100
|
-
// idle machine and stays comparable to the fixture tables above. Leaving it
|
|
101
|
-
// loaded would shift the verdict: the check reserves resident weight bytes
|
|
102
|
-
// through the fit margin, and the fitter also sees system-wide wired memory.
|
|
103
|
-
await unloadModel({ modelId: smallModelId, clearStorage: false });
|
|
104
|
-
watchVerdicts();
|
|
105
|
-
// 2. A load the fitter projects NOT to fit. The point of this example is that
|
|
106
|
-
// the SDK reports the verdict and then loads anyway — the check is
|
|
107
|
-
// evidence, not admission control.
|
|
108
|
-
if (!ATTEMPT_OVERSIZED) {
|
|
109
|
-
console.log('▸ Skipping the oversized gpt-oss-20B load.');
|
|
110
|
-
console.log('▸ Set QVAC_FIT_DEMO_ATTEMPT_OVERSIZED=1 to let it run and watch the');
|
|
111
|
-
console.log(' load proceed past a `does-not-fit` verdict (allocates ~11 GiB).');
|
|
112
|
-
}
|
|
113
|
-
else {
|
|
114
|
-
console.log('▸ Loading gpt-oss-20B @ 128k with an f32 KV cache');
|
|
115
|
-
console.log('▸ Expected verdict: projected NOT to fit');
|
|
116
|
-
console.log('▸ The load is attempted regardless. That is the fail-open contract:');
|
|
117
|
-
console.log(' the verdict is evidence, not admission control.\n');
|
|
118
|
-
const bigModelId = await loadModel({
|
|
119
|
-
modelSrc: GPT_OSS_20B_INST_Q4_K_M,
|
|
120
|
-
modelConfig: {
|
|
121
|
-
ctx_size: 131072,
|
|
122
|
-
'cache-type-k': 'f32',
|
|
123
|
-
'cache-type-v': 'f32'
|
|
124
|
-
}
|
|
125
|
-
});
|
|
126
|
-
console.log(`▸ Load returned ${bigModelId} — the advisory verdict did not block it`);
|
|
127
|
-
// Loading is not the same as being usable: a model can load and then fail
|
|
128
|
-
// at decode time. Gemma 4 31B does exactly that on this machine. So run a
|
|
129
|
-
// real completion and report throughput rather than trusting the load.
|
|
130
|
-
try {
|
|
131
|
-
const check = completion({
|
|
132
|
-
modelId: bigModelId,
|
|
133
|
-
history: [{ role: 'user', content: 'Name three colours. Answer briefly.' }],
|
|
134
|
-
stream: false,
|
|
135
|
-
generationParams: { predict: 40 }
|
|
136
|
-
});
|
|
137
|
-
const checkFinal = await check.final;
|
|
138
|
-
console.log(`▸ ...and it actually runs: ${checkFinal.stats?.tokensPerSecond?.toFixed(1) ?? '?'} tok/s ` +
|
|
139
|
-
`— the OS compressed and paged its way past the working-set budget`);
|
|
140
|
-
}
|
|
141
|
-
catch (inferenceError) {
|
|
142
|
-
console.log(`▸ ...but it cannot run: ${inferenceError instanceof Error ? inferenceError.message : String(inferenceError)}`);
|
|
143
|
-
console.log('▸ The verdict was right, and `loadModel` succeeding did not mean usable.');
|
|
144
|
-
}
|
|
145
|
-
await unloadModel({ modelId: bigModelId, clearStorage: false });
|
|
146
|
-
}
|
|
23
|
+
console.log('▸ Loaded — the fitter projected this one to fit here');
|
|
24
|
+
process.exit(0);
|
|
147
25
|
}
|
|
148
26
|
catch (error) {
|
|
149
|
-
// A failing load here is the native loader's own error, not the fit check.
|
|
150
|
-
// The check never throws and never converts a verdict into a load failure.
|
|
151
27
|
console.error('✖', error instanceof Error ? error.message : error);
|
|
152
|
-
process.
|
|
28
|
+
process.exit(1);
|
|
153
29
|
}
|
|
154
|
-
// The log subscription is an open stream and would otherwise keep the process
|
|
155
|
-
// alive after the work is done.
|
|
156
|
-
process.exit(process.exitCode ?? 0);
|