@qvac/sdk 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +194 -97
- package/README.md +29 -470
- package/dist/examples/asr/parakeet-sortformer.js +24 -14
- package/dist/examples/asr/parakeet-transcribepsy-filesystem.d.ts +2 -0
- package/dist/examples/asr/parakeet-transcribepsy-filesystem.js +105 -0
- package/dist/examples/asr/parakeet-unified-coreml.js +14 -9
- package/dist/examples/asr/whispercpp-prompt.js +9 -5
- package/dist/examples/assess-model-fit.d.ts +7 -9
- package/dist/examples/assess-model-fit.js +22 -43
- package/dist/examples/calc-tokens.d.ts +2 -0
- package/dist/examples/calc-tokens.js +21 -0
- package/dist/examples/classification/classify-image.js +11 -5
- package/dist/examples/completion-stop-reason.d.ts +3 -3
- package/dist/examples/completion-stop-reason.js +4 -4
- package/dist/examples/diffusion-flux3-klein.d.ts +2 -0
- package/dist/examples/diffusion-flux3-klein.js +102 -0
- package/dist/examples/diffusion-img2vid-ltx.js +7 -14
- package/dist/examples/diffusion-moss-image.d.ts +2 -0
- package/dist/examples/diffusion-moss-image.js +126 -0
- package/dist/examples/diffusion-qwen-image-21.d.ts +2 -0
- package/dist/examples/diffusion-qwen-image-21.js +115 -0
- package/dist/examples/diffusion-txt2vid-ltx.js +7 -14
- package/dist/examples/embed-multimodal.d.ts +2 -0
- package/dist/examples/embed-multimodal.js +80 -0
- package/dist/examples/embed-p2p.js +2 -2
- package/dist/examples/finetune/llamacpp-finetune.js +9 -4
- package/dist/examples/image/megaloc.d.ts +2 -0
- package/dist/examples/image/megaloc.js +37 -0
- package/dist/examples/llamacpp-multimodal.js +21 -1
- package/dist/examples/mcp-websearch.js +1 -1
- package/dist/examples/plugins.js +1 -1
- package/dist/examples/rag/rag-hyperdb/pipeline.js +2 -2
- package/dist/examples/rag/rag-turbovec.js +2 -2
- package/dist/examples/rerank.d.ts +2 -0
- package/dist/examples/rerank.js +33 -0
- package/dist/examples/sessions.d.ts +17 -0
- package/dist/examples/sessions.js +63 -0
- package/dist/examples/tools/llamacpp-deferred-tools.js +12 -8
- package/dist/examples/tools/llamacpp-tools-auto-dialect.js +4 -3
- package/dist/examples/tools/llamacpp-tools-gemma4.js +3 -3
- package/dist/examples/tools/llamacpp-tools-qwen35.js +4 -3
- package/dist/scripts/check-resource-collectors-packaging.js +7 -6
- package/dist/scripts/contract/build-contract.d.ts +2 -1
- package/dist/scripts/contract/build-contract.js +13 -1
- package/dist/scripts/contract/constants-registry.d.ts +19 -0
- package/dist/scripts/contract/constants-registry.js +9 -2
- package/dist/scripts/contract/method-shapes.d.ts +5 -0
- package/dist/scripts/contract/method-shapes.js +5 -0
- package/dist/scripts/enforce-inference-versions.d.ts +1 -1
- package/dist/scripts/enforce-inference-versions.js +28 -18
- package/dist/scripts/generate-bare-kit-runtimes.js +101 -69
- package/dist/src/client/api/assess-model-fit.d.ts +17 -17
- package/dist/src/client/api/assess-model-fit.js +19 -32
- package/dist/src/client/api/audio-edit.d.ts +5 -3
- package/dist/src/client/api/audio-edit.js +7 -5
- package/dist/src/client/api/audio-gen-result.js +136 -47
- package/dist/src/client/api/audio-gen.d.ts +40 -3
- package/dist/src/client/api/audio-gen.js +42 -5
- package/dist/src/client/api/audio-understand.d.ts +2 -2
- package/dist/src/client/api/audio-understand.js +4 -4
- package/dist/src/client/api/batch-completion.js +4 -3
- package/dist/src/client/api/bci-transcribe.js +3 -3
- package/dist/src/client/api/calc-tokens.d.ts +33 -0
- package/dist/src/client/api/calc-tokens.js +44 -0
- package/dist/src/client/api/cancel.d.ts +2 -2
- package/dist/src/client/api/cancel.js +3 -3
- package/dist/src/client/api/classify.d.ts +12 -5
- package/dist/src/client/api/classify.js +18 -6
- package/dist/src/client/api/client-request-id.d.ts +0 -15
- package/dist/src/client/api/client-request-id.js +15 -12
- package/dist/src/client/api/completion-stream.d.ts +9 -1
- package/dist/src/client/api/completion-stream.js +14 -3
- package/dist/src/client/api/decide.d.ts +1 -1
- package/dist/src/client/api/decide.js +3 -3
- package/dist/src/client/api/delete-cache.d.ts +9 -2
- package/dist/src/client/api/delete-cache.js +14 -4
- package/dist/src/client/api/diffusion.d.ts +38 -5
- package/dist/src/client/api/diffusion.js +42 -7
- package/dist/src/client/api/download-asset.js +3 -3
- package/dist/src/client/api/embed.d.ts +82 -29
- package/dist/src/client/api/embed.js +66 -21
- package/dist/src/client/api/finetune.js +3 -3
- package/dist/src/client/api/get-loaded-model-info.js +1 -1
- package/dist/src/client/api/get-model-info.d.ts +2 -2
- package/dist/src/client/api/get-model-info.js +3 -3
- package/dist/src/client/api/get-system-resources.d.ts +2 -2
- package/dist/src/client/api/get-system-resources.js +3 -3
- package/dist/src/client/api/heartbeat.d.ts +3 -2
- package/dist/src/client/api/heartbeat.js +4 -3
- package/dist/src/client/api/index.d.ts +8 -6
- package/dist/src/client/api/index.js +6 -4
- package/dist/src/client/api/invoke-plugin.d.ts +8 -2
- package/dist/src/client/api/invoke-plugin.js +18 -3
- package/dist/src/client/api/load-model.js +9 -6
- package/dist/src/client/api/logging-stream.d.ts +11 -2
- package/dist/src/client/api/logging-stream.js +12 -3
- package/dist/src/client/api/ocr.d.ts +5 -3
- package/dist/src/client/api/ocr.js +45 -22
- package/dist/src/client/api/rag.d.ts +1 -1
- package/dist/src/client/api/rag.js +3 -3
- package/dist/src/client/api/registry.d.ts +19 -8
- package/dist/src/client/api/registry.js +43 -17
- package/dist/src/client/api/rerank.d.ts +22 -0
- package/dist/src/client/api/rerank.js +39 -0
- package/dist/src/client/api/resume.d.ts +2 -1
- package/dist/src/client/api/resume.js +3 -3
- package/dist/src/client/api/rpc-server.d.ts +4 -4
- package/dist/src/client/api/rpc-server.js +12 -12
- package/dist/src/client/api/state.d.ts +2 -2
- package/dist/src/client/api/state.js +3 -3
- package/dist/src/client/api/subscribe-logs.d.ts +2 -2
- package/dist/src/client/api/subscribe-logs.js +4 -3
- package/dist/src/client/api/suspend.d.ts +4 -2
- package/dist/src/client/api/suspend.js +5 -4
- package/dist/src/client/api/text-to-speech.d.ts +14 -5
- package/dist/src/client/api/text-to-speech.js +86 -32
- package/dist/src/client/api/transcribe.d.ts +7 -3
- package/dist/src/client/api/transcribe.js +9 -3
- package/dist/src/client/api/translate.js +2 -2
- package/dist/src/client/api/unload-model.d.ts +19 -7
- package/dist/src/client/api/unload-model.js +31 -13
- package/dist/src/client/api/upscale.d.ts +6 -3
- package/dist/src/client/api/upscale.js +10 -5
- package/dist/src/client/api/vector-index.d.ts +51 -17
- package/dist/src/client/api/vector-index.js +101 -49
- package/dist/src/client/api/video-request.d.ts +5 -5
- package/dist/src/client/api/video.d.ts +3 -2
- package/dist/src/client/api/video.js +6 -5
- package/dist/src/client/api/vla.d.ts +9 -5
- package/dist/src/client/api/vla.js +17 -8
- package/dist/src/client/api/world-result.js +2 -2
- package/dist/src/client/api/world.d.ts +6 -6
- package/dist/src/client/api/world.js +6 -6
- package/dist/src/client/init-hooks.d.ts +6 -4
- package/dist/src/client/init-hooks.js +12 -15
- package/dist/src/client/logging-stream-registry.d.ts +6 -3
- package/dist/src/client/logging-stream-registry.js +36 -14
- package/dist/src/client/plugins-factory.d.ts +1 -1
- package/dist/src/client/plugins-factory.js +1 -1
- package/dist/src/client/rpc/bundle-stamp.d.ts +29 -0
- package/dist/src/client/rpc/bundle-stamp.js +75 -0
- package/dist/src/client/rpc/expo-home-dir.d.ts +2 -0
- package/dist/src/client/rpc/expo-home-dir.js +4 -0
- package/dist/src/client/rpc/expo-rpc-client.js +192 -138
- package/dist/src/client/rpc/node-rpc-client.js +324 -426
- package/dist/src/client/rpc/pear-runtime.d.ts +12 -0
- package/dist/src/client/rpc/pear-runtime.js +12 -0
- package/dist/src/client/rpc/rpc-client.d.ts +6 -4
- package/dist/src/client/rpc/rpc-client.js +168 -31
- package/dist/src/client/rpc/rpc-error.js +24 -2
- package/dist/src/client/rpc/unbundled-harness.d.ts +16 -0
- package/dist/src/client/rpc/unbundled-harness.js +27 -0
- package/dist/src/client/rpc/worker-failure.d.ts +27 -0
- package/dist/src/client/rpc/worker-failure.js +61 -0
- package/dist/src/client/session-registry.d.ts +75 -0
- package/dist/src/client/session-registry.js +149 -0
- package/dist/src/client/session.d.ts +125 -0
- package/dist/src/client/session.js +148 -0
- package/dist/src/commands/bundle/addons.d.ts +14 -0
- package/dist/src/commands/bundle/addons.js +65 -0
- package/dist/src/commands/bundle/constants.js +2 -0
- package/dist/src/commands/bundle/entry-gen.d.ts +1 -5
- package/dist/src/commands/bundle/entry-gen.js +18 -51
- package/dist/src/commands/bundle/index.d.ts +60 -14
- package/dist/src/commands/bundle/index.js +350 -126
- package/dist/src/commands/bundle/link.d.ts +58 -0
- package/dist/src/commands/bundle/link.js +135 -0
- package/dist/src/commands/bundle/node-support.d.ts +13 -0
- package/dist/src/commands/bundle/node-support.js +23 -0
- package/dist/src/commands/bundle/plugins.js +1 -1
- package/dist/src/commands/bundle/read-bundle.d.ts +9 -0
- package/dist/src/commands/bundle/read-bundle.js +27 -0
- package/dist/src/commands/bundle/resolve-sdk-import.d.ts +6 -0
- package/dist/src/commands/bundle/resolve-sdk-import.js +24 -28
- package/dist/src/commands/bundle/scratch.d.ts +35 -0
- package/dist/src/commands/bundle/scratch.js +178 -0
- package/dist/src/commands/bundle/targets.d.ts +14 -0
- package/dist/src/commands/bundle/targets.js +37 -0
- package/dist/src/commands/host-prebuilds/index.d.ts +8 -7
- package/dist/src/commands/host-prebuilds/index.js +13 -11
- package/dist/src/commands/index.d.ts +13 -12
- package/dist/src/commands/index.js +7 -7
- package/dist/src/commands/verify/abi.d.ts +21 -15
- package/dist/src/commands/verify/abi.js +12 -80
- package/dist/src/commands/verify/bare-kit-runtime.d.ts +20 -22
- package/dist/src/commands/verify/bare-kit-runtime.js +102 -150
- package/dist/src/commands/verify/bare-kit-runtimes.d.ts +10 -9
- package/dist/src/commands/verify/bare-kit-runtimes.js +21 -21
- package/dist/src/commands/verify/bundle-source.d.ts +1 -1
- package/dist/src/commands/verify/bundle-source.js +27 -23
- package/dist/src/commands/verify/engines-advice.d.ts +6 -12
- package/dist/src/commands/verify/engines-advice.js +9 -56
- package/dist/src/commands/verify/index.d.ts +34 -33
- package/dist/src/commands/verify/index.js +96 -204
- package/dist/src/commands/verify/node-modules-source.d.ts +1 -1
- package/dist/src/commands/verify/node-modules-source.js +1 -1
- package/dist/src/commands/verify/prebuilds.d.ts +4 -6
- package/dist/src/commands/verify/prebuilds.js +3 -4
- package/dist/src/commands/verify/resource-collectors.d.ts +0 -6
- package/dist/src/commands/verify/resource-collectors.js +3 -52
- package/dist/src/commands/verify/sidecar-runtime.d.ts +6 -0
- package/dist/src/commands/verify/sidecar-runtime.js +51 -0
- package/dist/src/expo/plugins/get-project-root.d.ts +1 -1
- package/dist/src/expo/plugins/withAndroidArchitecture.d.ts +1 -1
- package/dist/src/expo/plugins/withAndroidNdkVersion.d.ts +1 -1
- package/dist/src/expo/plugins/withAndroidNdkVersion.js +16 -10
- package/dist/src/expo/plugins/withBareKit.d.ts +59 -0
- package/dist/src/expo/plugins/withBareKit.js +204 -0
- package/dist/src/expo/plugins/withDeviceInfo.d.ts +1 -1
- package/dist/src/expo/plugins/withDeviceInfo.js +19 -5
- package/dist/src/expo/plugins/withMobileBundle.d.ts +5 -12
- package/dist/src/expo/plugins/withMobileBundle.js +38 -59
- package/dist/src/expo/plugins/withOpenCL.d.ts +1 -1
- package/dist/src/expo/plugins/withQvacSDK.d.ts +2 -1
- package/dist/src/expo/plugins/withQvacSDK.js +3 -0
- package/dist/src/index.d.ts +6 -5
- package/dist/src/index.js +10 -14
- package/dist/src/logging/addon.d.ts +1 -1
- package/dist/src/logging/base-logger.js +1 -1
- package/dist/src/logging/utils.js +11 -3
- package/dist/src/plugin-ids.d.ts +3 -1
- package/dist/src/plugin-ids.js +4 -0
- package/dist/src/schemas/public.d.ts +1 -1
- package/dist/src/schemas/public.js +1 -1
- package/dist/src/schemas/sdk-errors-client.d.ts +5 -6
- package/dist/src/schemas/sdk-errors-client.js +30 -36
- package/dist/src/schemas/sdk-errors-server.d.ts +12 -1
- package/dist/src/schemas/sdk-errors-server.js +65 -2
- package/dist/src/server/rpc/rpc-utils.d.ts +26 -0
- package/dist/src/server/rpc/rpc-utils.js +41 -0
- package/dist/src/utils/aggregate-events.d.ts +1 -1
- package/dist/src/utils/aggregate-events.js +2 -2
- package/dist/src/utils/errors-client.d.ts +37 -29
- package/dist/src/utils/errors-client.js +50 -41
- package/dist/src/utils/errors-server.d.ts +65 -1
- package/dist/src/utils/errors-server.js +122 -1
- package/dist/src/utils/mcp-adapter.d.ts +1 -1
- package/dist/src/utils/mcp-adapter.js +1 -1
- package/dist/src/utils/tool-helpers.js +1 -1
- package/dist/src/utils/withTimeout.js +34 -17
- package/dist/src/worker/handle-request.js +26 -21
- package/dist/src/worker/handler-utils.d.ts +2 -6
- package/dist/src/worker/handler-utils.js +2 -20
- package/dist/src/worker/index.d.ts +2 -4
- package/dist/src/worker/index.js +24 -34
- package/dist/src/worker/plugins/ggml-image-embedding.d.ts +2 -0
- package/dist/src/worker/plugins/ggml-image-embedding.js +1 -0
- package/dist/src/worker/plugins/llamacpp-rerank.d.ts +2 -0
- package/dist/src/worker/plugins/llamacpp-rerank.js +1 -0
- package/dist/src/worker/shim.d.ts +4 -0
- package/dist/src/worker/shim.js +7 -0
- package/dist/src/worker/start.d.ts +9 -0
- package/dist/src/worker/start.js +16 -0
- package/dist/src/worker/unbundled-shim.d.ts +2 -0
- package/dist/src/worker/unbundled-shim.js +4 -0
- package/docs/system-resources-support-matrix.md +27 -19
- package/package.json +91 -74
- package/src/electron-forge/index.cjs +65 -91
- package/dist/examples/model-fit-ui/server.d.ts +0 -24
- package/dist/examples/model-fit-ui/server.js +0 -221
- package/dist/src/client/rpc/ensure-worker-ready.d.ts +0 -9
- package/dist/src/client/rpc/ensure-worker-ready.js +0 -17
- package/dist/src/client/rpc/init-timeout.d.ts +0 -26
- package/dist/src/client/rpc/init-timeout.js +0 -25
- package/dist/src/client/rpc/worker-startup-error.d.ts +0 -12
- package/dist/src/client/rpc/worker-startup-error.js +0 -13
- package/dist/src/commands/bundle/bare-pack.d.ts +0 -21
- package/dist/src/commands/bundle/bare-pack.js +0 -77
- package/dist/src/commands/bundle/manifest.d.ts +0 -25
- package/dist/src/commands/bundle/manifest.js +0 -192
- package/dist/src/expo/plugins/link-addons.d.ts +0 -18
- package/dist/src/expo/plugins/link-addons.js +0 -65
- package/dist/src/pear/pre.d.ts +0 -21
- package/dist/src/pear/pre.js +0 -448
- package/dist/src/server/env.d.ts +0 -22
- package/dist/src/server/env.js +0 -59
- package/dist/src/server/utils/qvac-paths.d.ts +0 -2
- package/dist/src/server/utils/qvac-paths.js +0 -5
- package/dist/src/server/utils/worker-lock.d.ts +0 -3
- package/dist/src/server/utils/worker-lock.js +0 -67
- package/dist/src/worker/create-server.d.ts +0 -8
- package/dist/src/worker/create-server.js +0 -51
- package/dist/src/worker/lifecycle.d.ts +0 -23
- package/dist/src/worker/lifecycle.js +0 -149
package/README.md
CHANGED
|
@@ -20,63 +20,15 @@ For AI/LLM tools, use [https://docs.qvac.tether.io/llms-full.txt](https://docs.q
|
|
|
20
20
|
|
|
21
21
|
## Supported environments and installation
|
|
22
22
|
|
|
23
|
-
For mobile apps that use only raw PCM streaming transcription, list the required audio plugins and set `"includeAudioDecoder": false` in `qvac.config.*` before bundling or running Expo prebuild. You can also use `qvac bundle sdk --defer bare-ffmpeg`. Both options leave `bare-ffmpeg` out of the native addon manifest. Bundle FFmpeg when the app decodes compressed audio files or asks `audiogen-ggml` for a compressed output format; PCM and WAV output need no FFmpeg.
|
|
24
|
-
|
|
25
23
|
See https://docs.qvac.tether.io/sdk/getting-started/installation
|
|
26
24
|
|
|
27
|
-
On Node.js, the SDK
|
|
28
|
-
Its dependency range must satisfy `@qvac/inference`'s `engines.bare` requirement.
|
|
25
|
+
On Node.js, the SDK starts its own Bare worker with the Bare that `bare-sidecar` installs.
|
|
29
26
|
When upgrading the SDK, reinstall dependencies with the existing lockfile so the
|
|
30
|
-
package manager can replace an older, incompatible
|
|
31
|
-
or resolution that forces
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
## TTS 0.11 engines and memory fit
|
|
35
|
-
|
|
36
|
-
`@qvac/tts-ggml` 0.11 adds two desktop batch engines to `textToSpeech`.
|
|
37
|
-
Load MOSS-SoundEffect with `modelType: 'tts-ggml'` and
|
|
38
|
-
`modelConfig: { ttsEngine: 'moss-sfx' }`; the primary source is its single
|
|
39
|
-
GGUF. With an `AUDIOGEN_MOSS_SFX_V2_*` constant, pass its `.src` URL as
|
|
40
|
-
`modelSrc`, because the full descriptor selects the audiogen plugin. This also applies to fit assessment: use explicit `modelType` and `ttsEngine`, and a downloaded local file for metadata-only TTS fit. The bare `.src` URL lacks checksum/registry coordinates for a weightless stub; the audiogen descriptor/resource profile does not identify the TTS engine. Use
|
|
41
|
-
a text description, with optional `seconds`
|
|
42
|
-
(up to 30), `negativePrompt`, `steps`, `guidance` and `shift`. These controls
|
|
43
|
-
can also be load defaults. The output is 48 kHz.
|
|
44
|
-
|
|
45
|
-
Load MOSS-Speech with `modelConfig: { ttsEngine: 'moss-speech',
|
|
46
|
-
mossSpeechCodecModelSrc: <codec source> }` and the language-model GGUF as
|
|
47
|
-
`modelSrc`. Use either `text`, or `audio` plus
|
|
48
|
-
`sampleRate`. Audio is a JSON array of normalized mono samples in [-1, 1].
|
|
49
|
-
Earlier turns go in `messages`; each turn has a `role` and either `text` or
|
|
50
|
-
`audio` plus `sampleRate`. Per-call controls include `systemPrompt`,
|
|
51
|
-
`replyVoice` plus `replyVoiceSampleRate`, `maxReplySeconds`, `maxNewTokens`,
|
|
52
|
-
`textReply`, `greedy`, `temperature`, `topP` and `topK`. Consume `bufferStream` in the default streaming mode, or await `buffer` with
|
|
53
|
-
`stream: false`, then await `result.text` for any reply text.
|
|
54
|
-
Spoken mode can return an audio-only reply; use `textReply: true` when text is required.
|
|
55
|
-
Spoken replies are 24 kHz; `textReply: true` returns text with an empty audio
|
|
56
|
-
buffer. Omitting `maxNewTokens` preserves the addon's remaining-context budget.
|
|
57
|
-
Both engines run one native batch job. The default `stream: true` emits the completed audio as one chunk; `stream: false` collects it in `buffer`. Neither supports sentence streaming or duplex text input.
|
|
58
|
-
|
|
59
|
-
Standalone examples: [basic sound effects](examples/tts/moss-sound-effect.ts),
|
|
60
|
-
[sound-effect controls](examples/tts/moss-sound-effect-controls.ts),
|
|
61
|
-
[typed question to text reply](examples/tts/moss-speech.ts), and
|
|
62
|
-
[spoken question to spoken reply](examples/tts/moss-speech-audio.ts).
|
|
63
|
-
Run them from this package with `node --import tsx examples/tts/<file>.ts`;
|
|
64
|
-
the spoken-question example also takes a WAV file path. Each example downloads
|
|
65
|
-
its model, returns text or writes a WAV reply, unloads the model, and closes the SDK worker.
|
|
27
|
+
package manager can replace an older, incompatible `bare-sidecar`. Remove any override
|
|
28
|
+
or resolution that forces it below the SDK's range; installing a newer global `bare`
|
|
29
|
+
executable does not change the SDK's worker.
|
|
66
30
|
|
|
67
|
-
`
|
|
68
|
-
projections. Add `ttsWorkload` beside `modelConfig` on the assessment candidate:
|
|
69
|
-
`{ promptRows, referenceSamples, streaming }` for MOSS, or
|
|
70
|
-
`{ prompt, seconds }` for sound effects. MOSS needs its backbone and codec
|
|
71
|
-
decoder; reference encoding also needs the codec encoder. Prompt rows must
|
|
72
|
-
cover the complete native prompt, including dialogue and reference rows.
|
|
73
|
-
Missing workloads return an unknown assessment rather than guessing.
|
|
74
|
-
MOSS-Speech has no native fitter in this release and remains unknown. Workloads are fit-only; loading a MOSS model does not accept them and its automatic fit preflight remains unknown.
|
|
75
|
-
|
|
76
|
-
The CLI configuration picker includes both engines. Their conversation and
|
|
77
|
-
sound-effect request controls are library-only because the OpenAI speech
|
|
78
|
-
endpoint cannot represent those requests. Native performance and backend fixes
|
|
79
|
-
in the addon release are inherited without additional SDK controls.
|
|
31
|
+
For mobile apps that use only raw PCM streaming transcription, list the required audio plugins and set `"includeAudioDecoder": false` in `qvac.config.*` before bundling or running Expo prebuild. You can also use `qvac bundle sdk --defer bare-ffmpeg`. Both options leave `bare-ffmpeg` out of the worker bundle. Bundle FFmpeg when the app decodes compressed audio files or asks `audiogen-ggml` for a compressed output format; PCM and WAV output need no FFmpeg.
|
|
80
32
|
|
|
81
33
|
## Quickstart
|
|
82
34
|
|
|
@@ -131,160 +83,33 @@ try {
|
|
|
131
83
|
node quickstart.js
|
|
132
84
|
```
|
|
133
85
|
|
|
134
|
-
|
|
135
|
-
exceeds its budget (cold disk cache, antivirus scanning the native addon), the
|
|
136
|
-
call fails with `RPC_INIT_TIMEOUT`. Re-running usually succeeds; to give every
|
|
137
|
-
start more room, raise `rpcInitTimeoutMs` in `qvac.config.*` or set the
|
|
138
|
-
`QVAC_RPC_INIT_TIMEOUT_MS` environment variable.
|
|
139
|
-
|
|
140
|
-
## System resource diagnostics
|
|
141
|
-
|
|
142
|
-
Use `getSystemResources` to inspect locally observed CPU, system-memory, GPU, and
|
|
143
|
-
driver capabilities. Pass `sample: true` only when you also need a fresh usage
|
|
144
|
-
sample:
|
|
145
|
-
|
|
146
|
-
```ts
|
|
147
|
-
import { getSystemResources } from '@qvac/sdk'
|
|
148
|
-
|
|
149
|
-
const resources = await getSystemResources({ sample: true })
|
|
150
|
-
|
|
151
|
-
if (resources.capabilities.memory.totalBytes.status === 'supported') {
|
|
152
|
-
console.log('System memory:', resources.capabilities.memory.totalBytes.value)
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
if (resources.sample?.cpu.status === 'supported') {
|
|
156
|
-
console.log('CPU utilization:', resources.sample.cpu.value)
|
|
157
|
-
}
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
See the [system resources support matrix](./docs/system-resources-support-matrix.md)
|
|
161
|
-
for metric-level evidence and platform limitations.
|
|
162
|
-
|
|
163
|
-
Every metric reports `supported`, `unavailable`, `unverified`, or `failed`.
|
|
164
|
-
Supported values include provenance with a source and optional scope. These
|
|
165
|
-
values are diagnostics; they do not reserve memory or guarantee that a model
|
|
166
|
-
can be loaded.
|
|
167
|
-
|
|
168
|
-
GPU capabilities expose observed driver names, versions, and graphics APIs.
|
|
169
|
-
These observations do not prove that an inference backend is compatible.
|
|
170
|
-
|
|
171
|
-
Profiled inference operation events may include `event.backend` with the
|
|
172
|
-
selected backend and device, graphics API, driver, fallback reason, and probe
|
|
173
|
-
result. Addons attach backend metadata with `attachBackendDiagnostics`; the SDK
|
|
174
|
-
validates it before recording the operation event. `gpuId`, when present,
|
|
175
|
-
identifies a GPU from the current worker resource collector and is stable only
|
|
176
|
-
for that collector's lifetime. The SDK does not infer compatibility from driver
|
|
177
|
-
inventory or log text.
|
|
178
|
-
|
|
179
|
-
### Profiler resource gauges
|
|
180
|
-
|
|
181
|
-
Resource gauges are disabled by default. Enable them explicitly to attach one
|
|
182
|
-
worker resource sample to each profiled operation:
|
|
183
|
-
|
|
184
|
-
```ts
|
|
185
|
-
import { profiler } from '@qvac/sdk'
|
|
186
|
-
|
|
187
|
-
profiler.enable({ mode: 'verbose', includeResourceGauges: true })
|
|
188
|
-
|
|
189
|
-
// Run SDK operations, then inspect recentEvents[].resources.
|
|
190
|
-
const profile = profiler.exportJSON()
|
|
191
|
-
```
|
|
192
|
-
|
|
193
|
-
The sample uses the same status, provenance, and scope semantics as
|
|
194
|
-
`getSystemResources({ sample: true })`. Its `sampledAt` uses the same monotonic
|
|
195
|
-
clock as the profiling event's `ts`, so the two timestamps are comparable.
|
|
196
|
-
`resources.origin` records that the sample was taken on the `local` worker.
|
|
197
|
-
Samples are delivered to `profiler.onRecord`; they
|
|
198
|
-
are retained in `exportJSON().recentEvents` only in `verbose` mode. Enabling
|
|
199
|
-
gauges in `summary` mode still incurs the sampling cost without retaining them.
|
|
200
|
-
Disabling profiling or omitting `includeResourceGauges` performs no resource
|
|
201
|
-
sampling. Enabling gauges adds one CPU query and one query per GPU to each
|
|
202
|
-
profiled operation's response path. If the worker resource collector is not
|
|
203
|
-
initialized, the event omits the resource block.
|
|
204
|
-
|
|
205
|
-
## Pre-download model fit assessment
|
|
86
|
+
## Documentation
|
|
206
87
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
88
|
+
The [QVAC SDK documentation](https://docs.qvac.tether.io/sdk/) covers each
|
|
89
|
+
capability with its configuration and examples. See the
|
|
90
|
+
[API reference](https://docs.qvac.tether.io/sdk/reference/api/) for the full API
|
|
91
|
+
and the [release notes](https://docs.qvac.tether.io/sdk/reference/release-notes/)
|
|
92
|
+
for what changed in each release.
|
|
212
93
|
|
|
213
|
-
|
|
214
|
-
|
|
94
|
+
TranscribePsy EU-sub (`PARAKEET_TRANSCRIBEPSY_250M_Q8_0`, eight European
|
|
95
|
+
languages) has a separate Core ML distribution instead of a sidecar:
|
|
96
|
+
`PARAKEET_TRANSCRIBEPSY_250M_COREML_Q8_0` is an encoder-less GGUF whose
|
|
97
|
+
encoder and decoder bundles the SDK stages beside it, so both stages run on
|
|
98
|
+
the Neural Engine. It loads on macOS and iOS only; elsewhere the load is
|
|
99
|
+
refused up front with the ggml constant to use instead. See
|
|
100
|
+
[`examples/asr/parakeet-transcribepsy-filesystem.ts`](examples/asr/parakeet-transcribepsy-filesystem.ts),
|
|
101
|
+
which also shows the `longFormWindowFrames` / `longFormContextFrames` knobs
|
|
102
|
+
that bound the offline encoder window on long recordings.
|
|
215
103
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
policy: 'interactive-v1'
|
|
220
|
-
})
|
|
221
|
-
|
|
222
|
-
console.log(result.verdict) // 'likely-fits' | 'likely-too-large' | 'unknown'
|
|
223
|
-
```
|
|
224
|
-
|
|
225
|
-
The result is advisory: it does not block `loadModel`, reserve memory, or make a
|
|
226
|
-
performance claim. `unknown` is a real answer meaning the evidence does not
|
|
227
|
-
support a call either way — show it as "can't say", not as "no".
|
|
228
|
-
|
|
229
|
-
The assessment holds back a reserve of its own, beyond the headroom the engine's
|
|
230
|
-
fitter leaves, so a model the engine calls a fit can still come back
|
|
231
|
-
`likely-too-large` on a host with little room.
|
|
232
|
-
|
|
233
|
-
See [pre-download model fit assessment](./docs/assess-model-fit.md) for the
|
|
234
|
-
budget arithmetic, the reserve, and the supported engine and workload matrix.
|
|
235
|
-
|
|
236
|
-
## Streaming transcription statistics
|
|
237
|
-
|
|
238
|
-
Whisper and Parakeet duplex transcription sessions expose terminal engine
|
|
239
|
-
statistics after their event iterator completes:
|
|
240
|
-
|
|
241
|
-
```ts
|
|
242
|
-
const session = await transcribeStream({ modelId })
|
|
243
|
-
|
|
244
|
-
for await (const text of session) {
|
|
245
|
-
process.stdout.write(text)
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
const stats = await session.stats
|
|
249
|
-
console.log(stats?.audioDuration, stats?.realTimeFactor)
|
|
250
|
-
```
|
|
251
|
-
|
|
252
|
-
`session.stats` resolves to `undefined` when the engine does not report
|
|
253
|
-
statistics.
|
|
254
|
-
|
|
255
|
-
Batch `transcribe()` calls also expose terminal statistics:
|
|
256
|
-
|
|
257
|
-
```ts
|
|
258
|
-
const call = transcribe({ modelId, audioChunk })
|
|
259
|
-
const text = await call
|
|
260
|
-
const stats = await call.stats
|
|
261
|
-
console.log(text, stats?.encoderUsedCoreml)
|
|
262
|
-
```
|
|
263
|
-
|
|
264
|
-
`call.stats` resolves to `undefined` when the engine reports no statistics and
|
|
265
|
-
rejects if transcription fails.
|
|
266
|
-
|
|
267
|
-
## Parakeet Core ML encoders on Apple devices
|
|
268
|
-
|
|
269
|
-
Load a supported Parakeet registry constant as usual. On macOS and iOS, the SDK
|
|
270
|
-
downloads any complete, published Core ML encoder bundle with its GGUF and
|
|
271
|
-
places the `.mlmodelc` directory where the native addon discovers it. This is
|
|
272
|
-
available for TDT 0.6B v3, Unified English 0.6B, EOU 120M v1, and streaming
|
|
273
|
-
Sortformer v2.1. The download includes the sidecar weights, so first load takes
|
|
274
|
-
more time and disk space. Other platforms download only the GGUF. If the
|
|
275
|
-
sidecar is unavailable or cannot load, inference uses the GGUF encoder.
|
|
276
|
-
Sortformer v2.1 also needs a GGUF with the
|
|
277
|
-
`parakeet.model_variant=sortformer-streaming-v2.1-aosc` metadata. Caching both
|
|
278
|
-
sidecars alone does not activate Core ML. See
|
|
279
|
-
[`examples/asr/parakeet-unified-coreml.ts`](examples/asr/parakeet-unified-coreml.ts)
|
|
280
|
-
for a local check of a Unified GGUF and its encoder bundle.
|
|
104
|
+
See the [transcription documentation](https://docs.qvac.tether.io/sdk/ai-capabilities/transcription)
|
|
105
|
+
for the full TranscribePsy-250M-EU model (`PARAKEET_TRANSCRIBEPSY_250M_EU_Q8_0`)
|
|
106
|
+
and its `language` routes.
|
|
281
107
|
|
|
282
108
|
## Examples
|
|
283
109
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
2. Run with Node.js:
|
|
110
|
+
The [`examples`](./examples) directory has scripts covering every SDK capability.
|
|
111
|
+
To run one, set up the SDK from source as described in
|
|
112
|
+
[`CONTRIBUTING.md`](./CONTRIBUTING.md), then from `packages/sdk`:
|
|
288
113
|
|
|
289
114
|
```bash
|
|
290
115
|
# Compiled output
|
|
@@ -294,274 +119,8 @@ node dist/examples/path/to/example.js
|
|
|
294
119
|
npx tsx examples/path/to/example.ts
|
|
295
120
|
```
|
|
296
121
|
|
|
297
|
-
`examples/abot-world.ts` has a companion guide covering the hardware
|
|
298
|
-
requirements, scene-pack lifecycle, cancellation semantics and concurrency rules
|
|
299
|
-
of an interactive world session: see
|
|
300
|
-
[ABot-World interactive world sessions](./docs/abot-world.md).
|
|
301
|
-
|
|
302
|
-
## Build
|
|
303
|
-
|
|
304
|
-
Node.js 22 and npm are the only tools needed. No lockfile is committed; `package-lock.json` is gitignored.
|
|
305
|
-
|
|
306
|
-
Build and test against the in-repo engine at the same commit (this is what CI does):
|
|
307
|
-
|
|
308
|
-
```bash
|
|
309
|
-
npm run sdk-source:workspace
|
|
310
|
-
```
|
|
311
|
-
|
|
312
|
-
This installs and builds `../inference`, points `@qvac/inference` at it, and installs the SDK's dependencies. A plain `npm install` resolves `@qvac/inference` to its published release instead, which fails with `ERESOLVE` whenever the SDK on `main` already depends on addon versions the published engine has not been released against.
|
|
313
|
-
|
|
314
|
-
```bash
|
|
315
|
-
npm run build # or `watch` for hotreload
|
|
316
|
-
```
|
|
317
|
-
|
|
318
|
-
```bash
|
|
319
|
-
npm run build:pack
|
|
320
|
-
```
|
|
321
|
-
|
|
322
|
-
This outputs a tarball under `dist/qvac-sdk-{version}.tgz` that you can install in your project, e.g.:
|
|
323
|
-
|
|
324
|
-
```bash
|
|
325
|
-
npm i path/to/qvac-sdk-0.21.0.tgz
|
|
326
|
-
```
|
|
327
|
-
|
|
328
|
-
## Testing
|
|
329
|
-
|
|
330
|
-
The SDK test suite is organized into two buckets by runtime:
|
|
331
|
-
|
|
332
|
-
| Bucket | Runtime | Location | Command |
|
|
333
|
-
| ----------------- | --------- | -------- | -------------------------------------- |
|
|
334
|
-
| Unit | Node | `test/` | `npm run test:unit` |
|
|
335
|
-
| Client (consumer) | Node / RN | `e2e/` | See [`e2e/README.md`](./e2e/README.md) |
|
|
336
|
-
|
|
337
|
-
See [`TESTING.md`](./TESTING.md) for the full decision tree on where new tests should land.
|
|
338
|
-
|
|
339
122
|
## Contributing
|
|
340
123
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
#### Format
|
|
346
|
-
|
|
347
|
-
**Commit messages:**
|
|
348
|
-
|
|
349
|
-
```
|
|
350
|
-
prefix[tags]?: subject
|
|
351
|
-
```
|
|
352
|
-
|
|
353
|
-
**PR titles:**
|
|
354
|
-
|
|
355
|
-
```
|
|
356
|
-
TICKET prefix[tags]: subject
|
|
357
|
-
```
|
|
358
|
-
|
|
359
|
-
#### Allowed Prefixes
|
|
360
|
-
|
|
361
|
-
- `feat` - New features or capabilities
|
|
362
|
-
- `fix` - Bug fixes
|
|
363
|
-
- `doc` - Documentation changes
|
|
364
|
-
- `test` - Test additions or modifications
|
|
365
|
-
- `mod` - Model-related changes
|
|
366
|
-
- `chore` - Maintenance tasks
|
|
367
|
-
- `infra` - CI/CD, tooling, infrastructure
|
|
368
|
-
|
|
369
|
-
#### Allowed Tags
|
|
370
|
-
|
|
371
|
-
Tags are optional:
|
|
372
|
-
|
|
373
|
-
- `[api]` - API changes (non-breaking)
|
|
374
|
-
- `[bc]` - Breaking changes (including breaking API changes)
|
|
375
|
-
|
|
376
|
-
#### Examples
|
|
377
|
-
|
|
378
|
-
**Valid commit messages:**
|
|
379
|
-
|
|
380
|
-
```bash
|
|
381
|
-
feat: add RAG support for LanceDB
|
|
382
|
-
fix[api]: fix completion stream error handling
|
|
383
|
-
doc: update installation instructions
|
|
384
|
-
feat[bc]: redesign loadModel signature
|
|
385
|
-
chore: update dependencies
|
|
386
|
-
```
|
|
387
|
-
|
|
388
|
-
**Valid PR titles:**
|
|
389
|
-
|
|
390
|
-
```bash
|
|
391
|
-
QVAC-123 feat: add RAG support for LanceDB
|
|
392
|
-
QVAC-456 fix[api]: fix completion stream error handling
|
|
393
|
-
QVAC-789 doc: update installation instructions
|
|
394
|
-
QVAC-101 feat[bc]: redesign loadModel signature
|
|
395
|
-
```
|
|
396
|
-
|
|
397
|
-
#### Code Examples Requirements
|
|
398
|
-
|
|
399
|
-
When creating PRs with specific tags, you must include code examples in the PR description:
|
|
400
|
-
|
|
401
|
-
**`[bc]` tag requirements:**
|
|
402
|
-
|
|
403
|
-
Must include BEFORE/AFTER code examples showing the migration path:
|
|
404
|
-
|
|
405
|
-
````markdown
|
|
406
|
-
## BC Changes
|
|
407
|
-
|
|
408
|
-
**BEFORE:**
|
|
409
|
-
|
|
410
|
-
```typescript
|
|
411
|
-
const model = await loadModel('model-path')
|
|
412
|
-
```
|
|
413
|
-
|
|
414
|
-
**AFTER:**
|
|
415
|
-
|
|
416
|
-
```typescript
|
|
417
|
-
const modelId = await loadModel('model-path', { modelType: 'llm' })
|
|
418
|
-
```
|
|
419
|
-
````
|
|
420
|
-
|
|
421
|
-
Or using inline comments:
|
|
422
|
-
|
|
423
|
-
````markdown
|
|
424
|
-
```typescript
|
|
425
|
-
// old
|
|
426
|
-
const model = await loadModel('model-path')
|
|
427
|
-
|
|
428
|
-
// new
|
|
429
|
-
const modelId = await loadModel('model-path', { modelType: 'llm' })
|
|
430
|
-
```
|
|
431
|
-
````
|
|
432
|
-
|
|
433
|
-
**`[api]` tag requirements (non-breaking):**
|
|
434
|
-
|
|
435
|
-
Must include at least one fenced code block showing the new API usage:
|
|
436
|
-
|
|
437
|
-
````markdown
|
|
438
|
-
## New API
|
|
439
|
-
|
|
440
|
-
```typescript
|
|
441
|
-
// New completion API with streaming support
|
|
442
|
-
for await (const token of completion({
|
|
443
|
-
modelId,
|
|
444
|
-
history: [{ role: 'user', content: 'Hello!' }]
|
|
445
|
-
}).tokenStream) {
|
|
446
|
-
process.stdout.write(token)
|
|
447
|
-
}
|
|
448
|
-
```
|
|
449
|
-
````
|
|
450
|
-
|
|
451
|
-
#### Validation
|
|
452
|
-
|
|
453
|
-
- **Commit messages** are validated automatically via Husky commit-msg hook
|
|
454
|
-
- **PR titles and descriptions** are validated via GitHub Actions on PR creation/update
|
|
455
|
-
- Invalid commits or PRs will be rejected with helpful error messages
|
|
456
|
-
- **Auto-skipped commits:** The following Git-generated commits bypass validation:
|
|
457
|
-
- Merge commits (e.g., `Merge pull request #123`)
|
|
458
|
-
- Version bumps (e.g., `1.0.0`, `v1.0.0`)
|
|
459
|
-
- Revert commits (e.g., `Revert "feat: add feature"`)
|
|
460
|
-
- Squash commits (e.g., `squash! fix: bug fix`)
|
|
461
|
-
|
|
462
|
-
#### Generating Changelogs
|
|
463
|
-
|
|
464
|
-
Once your PRs are merged into `dev`, you can generate a changelog:
|
|
465
|
-
|
|
466
|
-
```bash
|
|
467
|
-
npm run changelog:generate
|
|
468
|
-
```
|
|
469
|
-
|
|
470
|
-
This will:
|
|
471
|
-
|
|
472
|
-
1. Compare versions between `dev` and `main` branches
|
|
473
|
-
2. Collect all merged PRs
|
|
474
|
-
3. Parse and classify each PR by prefix
|
|
475
|
-
4. Generate `changelog/<version>/CHANGELOG.md`
|
|
476
|
-
5. Generate `changelog/<version>/breaking.md` for BC changes (with code examples)
|
|
477
|
-
6. Generate `changelog/<version>/api.md` for API changes (with code examples)
|
|
478
|
-
|
|
479
|
-
## Speaker-aware transcription
|
|
480
|
-
|
|
481
|
-
ASR 0.8 adds `MODEL_TYPES.mossTranscribe` and the
|
|
482
|
-
`@qvac/sdk/moss-transcribe/plugin` entry point. MOSS processes a whole recording;
|
|
483
|
-
use `transcribe`, since it has no duplex streaming or configuration reload.
|
|
484
|
-
|
|
485
|
-
```ts
|
|
486
|
-
import { loadModel, transcribe, MOSS_TRANSCRIBE_DIARIZE_Q8_0 } from '@qvac/sdk'
|
|
487
|
-
|
|
488
|
-
const modelId = await loadModel({
|
|
489
|
-
modelSrc: MOSS_TRANSCRIBE_DIARIZE_Q8_0,
|
|
490
|
-
modelConfig: { useGPU: true, maxThreads: 4 }
|
|
491
|
-
})
|
|
492
|
-
const segments = await transcribe({
|
|
493
|
-
modelId,
|
|
494
|
-
audioChunk: '/recordings/meeting.wav',
|
|
495
|
-
metadata: true,
|
|
496
|
-
hotwords: ['QVAC'],
|
|
497
|
-
maxNewTokens: 2048
|
|
498
|
-
})
|
|
499
|
-
```
|
|
500
|
-
|
|
501
|
-
For a runnable example with download progress and resource cleanup, see
|
|
502
|
-
[the MOSS file transcription example](examples/asr/moss-transcribe-filesystem.ts).
|
|
503
|
-
From `packages/sdk`, run:
|
|
504
|
-
|
|
505
|
-
```bash
|
|
506
|
-
npx tsx examples/asr/moss-transcribe-filesystem.ts examples/audio/diarization-sample-16k.wav
|
|
507
|
-
# Use local weights and per-request hotwords:
|
|
508
|
-
npx tsx examples/asr/moss-transcribe-filesystem.ts meeting.wav ./moss.gguf QVAC OpenMOSS
|
|
509
|
-
```
|
|
510
|
-
|
|
511
|
-
Segments retain `speaker` labels such as `S01`, zero-based `speakerId`, and
|
|
512
|
-
`startMs`/`endMs`. Hotwords apply to each request: at most 64 nonempty terms of
|
|
513
|
-
64 UTF-8 bytes each. A MOSS `prompt` replaces the default instruction and cannot
|
|
514
|
-
be combined with hotwords. `maxNewTokens: 0` uses the model default. These two
|
|
515
|
-
MOSS-specific options are rejected for Whisper and Parakeet models.
|
|
516
|
-
|
|
517
|
-
The published addon contains Whisper v1.9.4, so existing Whisper calls keep
|
|
518
|
-
their API and use the updated backend. MOSS memory-fit assessment projects the
|
|
519
|
-
declared recording and decoding options without loading weights:
|
|
520
|
-
|
|
521
|
-
```typescript
|
|
522
|
-
const fit = await assessModelFit({
|
|
523
|
-
models: [
|
|
524
|
-
{
|
|
525
|
-
modelSrc: MOSS_TRANSCRIBE_DIARIZE_Q8_0,
|
|
526
|
-
modelConfig: { useGPU: false },
|
|
527
|
-
transcriptionWorkload: { audioSeconds: 300, hotwords: ['Erin'], maxNewTokens: 0 }
|
|
528
|
-
}
|
|
529
|
-
]
|
|
530
|
-
})
|
|
531
|
-
```
|
|
532
|
-
|
|
533
|
-
Import `assessModelFit` from `@qvac/sdk`. Use the longest recording you intend
|
|
534
|
-
to transcribe, with the same prompt/hotwords and token limit as the request.
|
|
535
|
-
Without `transcriptionWorkload`, MOSS has no native fit evidence, its recording
|
|
536
|
-
window is omitted, and the result can remain `unknown`. Assessment uses a registry description when available,
|
|
537
|
-
or a local model path; it downloads no full weights. MOSS CoreML requires an
|
|
538
|
-
addon revision newer than the published 0.8.0 release.
|
|
539
|
-
|
|
540
|
-
The SDK and inference use the published `@qvac/asr-ggml` package at `^0.8.0`.
|
|
541
|
-
|
|
542
|
-
## Nemotron 3 Diarization
|
|
543
|
-
|
|
544
|
-
Load a local Nemotron 3 Diarization GGUF with `modelType: 'parakeet-transcription'`.
|
|
545
|
-
The addon detects the variant from the GGUF and supports up to eight speakers,
|
|
546
|
-
including overlapping activity. This model reports speaker turns rather than words.
|
|
547
|
-
|
|
548
|
-
Use `transcribe({ modelId, audioChunk, metadata: true })` for offline diarization.
|
|
549
|
-
Each result can include `speakerSegments` with zero-based `speakerId`, `startMs`,
|
|
550
|
-
and `endMs`. Load with `modelConfig.streaming: true` before using `transcribeStream`.
|
|
551
|
-
Streaming metadata carries `speakerId` on each diarization segment.
|
|
552
|
-
For streaming speaker-activity events, enable `streamingSpeakerVad` at load or
|
|
553
|
-
`emitSpeakerVad` under `parakeetStreamingConfig` per `transcribeStream` call;
|
|
554
|
-
conversation VAD events retain their dominant `speakerId`.
|
|
555
|
-
|
|
556
|
-
`diarizationThreshold` (0–1) and `diarizationMinSegmentMs` (nonnegative integer)
|
|
557
|
-
are supported in `modelConfig` and per-call `parakeetStreamingConfig`. Omit them
|
|
558
|
-
to preserve model defaults. Nemotron defaults to a 0.5 threshold and 200 ms minimum
|
|
559
|
-
turn; its default left context is 0 ms, while an explicitly configured 80 ms is
|
|
560
|
-
preserved. CLI `verbose_json` exposes offline turns as `speaker_segments` with
|
|
561
|
-
`speaker_id` and start/end timestamps in seconds.
|
|
562
|
-
|
|
563
|
-
See [the local Nemotron diarization example](examples/asr/nemotron-diarization-filesystem.ts):
|
|
564
|
-
|
|
565
|
-
```bash
|
|
566
|
-
npx tsx examples/asr/nemotron-diarization-filesystem.ts ./nemotron-diarization.gguf meeting.wav
|
|
567
|
-
```
|
|
124
|
+
See [`CONTRIBUTING.md`](./CONTRIBUTING.md) for building from source, commit and
|
|
125
|
+
PR title format, and changelog generation, and [`TESTING.md`](./TESTING.md) for
|
|
126
|
+
where new tests belong.
|
|
@@ -2,27 +2,28 @@
|
|
|
2
2
|
* Parakeet Sortformer diarization + TDT transcription pipeline.
|
|
3
3
|
*
|
|
4
4
|
* Usage:
|
|
5
|
-
* npx tsx examples/asr/parakeet-sortformer.ts [sortformer-gguf]
|
|
5
|
+
* npx tsx examples/asr/parakeet-sortformer.ts <wav-file> [sortformer-gguf]
|
|
6
6
|
*
|
|
7
7
|
* Two-step flow: Sortformer v2.1 diarizes the audio, then TDT transcribes each
|
|
8
|
-
* speaker segment. Defaults to registry GGUFs
|
|
9
|
-
* `examples/audio/diarization-sample-16k.wav`. For live streaming + AOSC, see
|
|
8
|
+
* speaker segment. Defaults to registry GGUFs. For live streaming + AOSC, see
|
|
10
9
|
* `parakeet-sortformer-streaming.ts`.
|
|
11
10
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* Audio should be 16 kHz mono 16-bit PCM in a WAV container. A multi-speaker
|
|
12
|
+
* sample is in the QVAC source repo but not the published npm package:
|
|
14
13
|
* https://github.com/tetherto/qvac/blob/main/packages/sdk/examples/audio/diarization-sample-16k.wav
|
|
15
14
|
*/
|
|
16
15
|
import { loadModel, unloadModel, transcribe, PARAKEET_TDT_0_6B_V3_Q8_0, PARAKEET_SORTFORMER_4SPK_V2_1_Q8_0 } from '@qvac/sdk';
|
|
17
|
-
import {
|
|
18
|
-
import { fileURLToPath } from 'url';
|
|
16
|
+
import { join } from 'path';
|
|
19
17
|
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
20
18
|
import { tmpdir } from 'os';
|
|
21
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
22
19
|
const args = process.argv.slice(2);
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
20
|
+
if (!args[0]) {
|
|
21
|
+
console.error('Usage: npx tsx examples/asr/parakeet-sortformer.ts <wav-file> [sortformer-gguf]');
|
|
22
|
+
console.error('\nIf the model path is omitted, defaults to the registry model.');
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
const audioFilePath = args[0];
|
|
26
|
+
const sortformerSrc = args[1] ?? PARAKEET_SORTFORMER_4SPK_V2_1_Q8_0;
|
|
26
27
|
try {
|
|
27
28
|
// ── Step 1: Diarize with Sortformer ──
|
|
28
29
|
const sfModelId = await loadModel({
|
|
@@ -71,15 +72,24 @@ catch (error) {
|
|
|
71
72
|
process.exit(1);
|
|
72
73
|
}
|
|
73
74
|
// ── Helpers ──
|
|
75
|
+
// Sortformer reports each segment as `Speaker N: HH:MM:SS.mmm - HH:MM:SS.mmm`
|
|
74
76
|
function parseDiarization(text) {
|
|
75
77
|
const segs = [];
|
|
76
78
|
for (const line of text.split('\n')) {
|
|
77
|
-
const m = line.match(/Speaker (\d+): ([\d.]+)
|
|
78
|
-
if (m)
|
|
79
|
-
segs.push({
|
|
79
|
+
const m = line.match(/Speaker (\d+): (\d+):(\d+):([\d.]+) - (\d+):(\d+):([\d.]+)/);
|
|
80
|
+
if (m) {
|
|
81
|
+
segs.push({
|
|
82
|
+
speaker: +m[1],
|
|
83
|
+
start: seconds(m[2], m[3], m[4]),
|
|
84
|
+
end: seconds(m[5], m[6], m[7])
|
|
85
|
+
});
|
|
86
|
+
}
|
|
80
87
|
}
|
|
81
88
|
return segs.sort((a, b) => a.start - b.start);
|
|
82
89
|
}
|
|
90
|
+
function seconds(h, m, s) {
|
|
91
|
+
return +h * 3600 + +m * 60 + +s;
|
|
92
|
+
}
|
|
83
93
|
function readPcm(wavPath) {
|
|
84
94
|
const buf = readFileSync(wavPath);
|
|
85
95
|
const dataOffset = buf.indexOf('data') + 4;
|