@qvac/sdk 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +15 -68
- package/dist/client/api/cancel.d.ts +1 -1
- package/dist/client/api/cancel.js +1 -1
- package/dist/client/api/completion-stream.d.ts +51 -12
- package/dist/client/api/completion-stream.js +129 -56
- package/dist/client/api/delete-cache.d.ts +1 -0
- package/dist/client/api/delete-cache.js +1 -0
- package/dist/client/api/diffusion.d.ts +17 -2
- package/dist/client/api/diffusion.js +21 -4
- package/dist/client/api/embed.d.ts +4 -0
- package/dist/client/api/finetune.d.ts +21 -0
- package/dist/client/api/get-loaded-model-info.d.ts +25 -0
- package/dist/client/api/get-loaded-model-info.js +36 -0
- package/dist/client/api/get-model-info.d.ts +9 -1
- package/dist/client/api/get-model-info.js +8 -0
- package/dist/client/api/heartbeat.d.ts +2 -2
- package/dist/client/api/heartbeat.js +2 -2
- package/dist/client/api/index.d.ts +3 -1
- package/dist/client/api/index.js +3 -1
- package/dist/client/api/invoke-plugin.d.ts +16 -0
- package/dist/client/api/invoke-plugin.js +16 -0
- package/dist/client/api/load-model.d.ts +36 -3
- package/dist/client/api/load-model.js +31 -10
- package/dist/client/api/provide.d.ts +11 -5
- package/dist/client/api/provide.js +11 -6
- package/dist/client/api/rag.d.ts +9 -0
- package/dist/client/api/rag.js +9 -0
- package/dist/client/api/registry.d.ts +26 -0
- package/dist/client/api/registry.js +26 -0
- package/dist/client/api/resume.d.ts +20 -4
- package/dist/client/api/resume.js +20 -4
- package/dist/client/api/state.d.ts +23 -0
- package/dist/client/api/state.js +29 -0
- package/dist/client/api/stop-provider.d.ts +11 -5
- package/dist/client/api/stop-provider.js +11 -5
- package/dist/client/api/suspend.d.ts +21 -3
- package/dist/client/api/suspend.js +21 -3
- package/dist/client/api/text-to-speech.d.ts +64 -6
- package/dist/client/api/text-to-speech.js +423 -47
- package/dist/client/api/transcribe.d.ts +29 -4
- package/dist/client/api/transcribe.js +86 -28
- package/dist/client/api/translate.d.ts +1 -0
- package/dist/client/api/translate.js +1 -0
- package/dist/client/rpc/bare-client.js +22 -1
- package/dist/client/rpc/expo-rpc-client.d.ts +1 -1
- package/dist/client/rpc/expo-rpc-client.js +96 -4
- package/dist/client/rpc/node-rpc-client.js +36 -0
- package/dist/examples/audio/mic-input.d.ts +9 -0
- package/dist/examples/audio/mic-input.js +111 -0
- package/dist/examples/completion-events.d.ts +18 -0
- package/dist/examples/completion-events.js +79 -0
- package/dist/examples/delegated-inference/composite.js +12 -18
- package/dist/examples/delegated-inference/consumer-profiled.js +4 -11
- package/dist/examples/delegated-inference/consumer.js +8 -13
- package/dist/examples/delegated-inference/provider.js +11 -16
- package/dist/examples/diffusion-flux2-klein-img2img.d.ts +2 -0
- package/dist/examples/diffusion-flux2-klein-img2img.js +60 -0
- package/dist/examples/diffusion-flux2-klein.js +1 -0
- package/dist/examples/diffusion-img2img.d.ts +2 -0
- package/dist/examples/diffusion-img2img.js +49 -0
- package/dist/examples/diffusion-txt2img.js +2 -1
- package/dist/examples/kv-cache-example.js +11 -12
- package/dist/examples/llamacpp-cache.d.ts +2 -0
- package/dist/examples/llamacpp-cache.js +107 -0
- package/dist/examples/llamacpp-dynamic-tools.d.ts +2 -0
- package/dist/examples/llamacpp-dynamic-tools.js +144 -0
- package/dist/examples/llamacpp-native-tools.js +2 -8
- package/dist/examples/llamacpp-structured-output.d.ts +2 -0
- package/dist/examples/llamacpp-structured-output.js +106 -0
- package/dist/examples/llamacpp-tools-pythonic.d.ts +2 -0
- package/dist/examples/llamacpp-tools-pythonic.js +69 -0
- package/dist/examples/plugins.js +4 -4
- package/dist/examples/suspend-resume.js +21 -4
- package/dist/examples/transcription/parakeet-microphone-record.js +6 -33
- package/dist/examples/transcription/whispercpp-filesystem-streaming.js +7 -5
- package/dist/examples/transcription/whispercpp-filesystem.js +12 -3
- package/dist/examples/transcription/whispercpp-microphone-record.js +6 -29
- package/dist/examples/translation/translation-stream.js +5 -5
- package/dist/examples/tts/llm-to-tts-streaming.d.ts +16 -0
- package/dist/examples/tts/llm-to-tts-streaming.js +143 -0
- package/dist/examples/tts/supertonic-multilingual.js +3 -3
- package/dist/examples/tts/supertonic.js +3 -3
- package/dist/examples/tts/utils.d.ts +11 -1
- package/dist/examples/tts/utils.js +70 -17
- package/dist/examples/voice-assistant/voice-assistant.d.ts +2 -0
- package/dist/examples/voice-assistant/voice-assistant.js +207 -0
- package/dist/expo/plugins/withMobileBundle.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/logging/logger.d.ts +11 -0
- package/dist/logging/logger.js +11 -0
- package/dist/logging/types.d.ts +18 -0
- package/dist/models/registry/models.d.ts +6620 -6185
- package/dist/models/registry/models.js +6296 -5318
- package/dist/models/update-models/codegen.js +84 -23
- package/dist/models/update-models/companions.d.ts +18 -0
- package/dist/models/update-models/companions.js +175 -0
- package/dist/models/update-models/naming.js +0 -25
- package/dist/models/update-models/registry.js +4 -2
- package/dist/models/update-models/types.d.ts +37 -9
- package/dist/profiling/index.d.ts +49 -0
- package/dist/profiling/index.js +49 -0
- package/dist/schemas/cancel.d.ts +6 -2
- package/dist/schemas/cancel.js +14 -7
- package/dist/schemas/common.d.ts +361 -279
- package/dist/schemas/common.js +26 -5
- package/dist/schemas/completion-event.d.ts +225 -0
- package/dist/schemas/completion-event.js +79 -0
- package/dist/schemas/completion-stream.d.ts +122 -31
- package/dist/schemas/completion-stream.js +137 -36
- package/dist/schemas/delegate.d.ts +0 -2
- package/dist/schemas/delegate.js +25 -6
- package/dist/schemas/download-asset.d.ts +5 -5
- package/dist/schemas/embed.js +21 -9
- package/dist/schemas/engine-addon-map.d.ts +11 -1
- package/dist/schemas/engine-addon-map.js +2 -2
- package/dist/schemas/finetune.d.ts +16 -16
- package/dist/schemas/finetune.js +8 -8
- package/dist/schemas/get-loaded-model-info.d.ts +52 -0
- package/dist/schemas/get-loaded-model-info.js +44 -0
- package/dist/schemas/get-model-info.d.ts +2 -2
- package/dist/schemas/get-model-info.js +103 -32
- package/dist/schemas/heartbeat.d.ts +0 -1
- package/dist/schemas/index.d.ts +4 -1
- package/dist/schemas/index.js +4 -1
- package/dist/schemas/llamacpp-config.d.ts +14 -4
- package/dist/schemas/llamacpp-config.js +5 -0
- package/dist/schemas/load-model.d.ts +3233 -1260
- package/dist/schemas/load-model.js +23 -16
- package/dist/schemas/model-src-utils.d.ts +7 -6
- package/dist/schemas/model-src-utils.js +27 -1
- package/dist/schemas/model-types.d.ts +4 -4
- package/dist/schemas/ocr.d.ts +1 -1
- package/dist/schemas/plugin.d.ts +27 -3
- package/dist/schemas/plugin.js +27 -2
- package/dist/schemas/profiling.js +11 -3
- package/dist/schemas/provide.d.ts +1 -3
- package/dist/schemas/provide.js +0 -3
- package/dist/schemas/rag.d.ts +4 -4
- package/dist/schemas/registry.d.ts +10 -10
- package/dist/schemas/registry.js +36 -14
- package/dist/schemas/reload-config.d.ts +73 -7
- package/dist/schemas/sdcpp-config.d.ts +16 -8
- package/dist/schemas/sdcpp-config.js +118 -25
- package/dist/schemas/sdk-config.d.ts +116 -90
- package/dist/schemas/sdk-config.js +16 -0
- package/dist/schemas/sdk-errors-client.d.ts +3 -0
- package/dist/schemas/sdk-errors-client.js +16 -1
- package/dist/schemas/sdk-errors-server.d.ts +3 -0
- package/dist/schemas/sdk-errors-server.js +23 -0
- package/dist/schemas/state.d.ts +23 -0
- package/dist/schemas/state.js +14 -0
- package/dist/schemas/stop-provide.d.ts +0 -5
- package/dist/schemas/stop-provide.js +2 -6
- package/dist/schemas/text-to-speech.d.ts +93 -27
- package/dist/schemas/text-to-speech.js +27 -0
- package/dist/schemas/tools.d.ts +16 -0
- package/dist/schemas/tools.js +15 -0
- package/dist/schemas/transcription-config.d.ts +8 -8
- package/dist/schemas/transcription.d.ts +33 -0
- package/dist/schemas/transcription.js +9 -0
- package/dist/schemas/translate.js +59 -19
- package/dist/schemas/translation-config.d.ts +157 -133
- package/dist/schemas/translation-config.js +34 -11
- package/dist/server/bare/delegate-rpc-client.d.ts +1 -1
- package/dist/server/bare/delegate-rpc-client.js +131 -122
- package/dist/server/bare/hyperswarm.d.ts +2 -2
- package/dist/server/bare/hyperswarm.js +12 -7
- package/dist/server/bare/ops/cancel.js +7 -0
- package/dist/server/bare/ops/kv-cache-utils.d.ts +7 -10
- package/dist/server/bare/ops/kv-cache-utils.js +31 -40
- package/dist/server/bare/ops/load-model.js +0 -1
- package/dist/server/bare/ops/transcribe.d.ts +11 -4
- package/dist/server/bare/ops/transcribe.js +25 -4
- package/dist/server/bare/ops/translate.js +42 -4
- package/dist/server/bare/ops/unload-model.js +21 -19
- package/dist/server/bare/plugins/llamacpp-completion/ops/cache-logger.js +1 -2
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +13 -8
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +257 -85
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts +101 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js +135 -0
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +83 -26
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +57 -32
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +10 -16
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +69 -44
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +113 -94
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +1 -1
- package/dist/server/bare/plugins/onnx-ocr/plugin.js +1 -2
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech-stream.js +116 -0
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech.d.ts +2 -3
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech.js +71 -9
- package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +41 -13
- package/dist/server/bare/plugins/onnx-tts/plugin.js +48 -3
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +15 -7
- package/dist/server/bare/plugins/parakeet-transcription/plugin.js +5 -6
- package/dist/server/bare/plugins/sdcpp-generation/ops/diffusion.js +6 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +8 -5
- package/dist/server/bare/plugins/sdcpp-generation/plugin.js +13 -13
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +17 -1
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +40 -17
- package/dist/server/bare/registry/config-registry.js +14 -3
- package/dist/server/bare/registry/model-registry.d.ts +8 -10
- package/dist/server/bare/registry/model-registry.js +17 -12
- package/dist/server/bare/registry/registry-client.js +79 -19
- package/dist/server/bare/runtime-lifecycle.d.ts +2 -1
- package/dist/server/bare/runtime-lifecycle.js +13 -2
- package/dist/server/bare/types/addon-responses.d.ts +8 -3
- package/dist/server/bare/utils/transcribe-metadata.d.ts +21 -0
- package/dist/server/bare/utils/transcribe-metadata.js +28 -0
- package/dist/server/bare/utils/tts-stats.d.ts +22 -0
- package/dist/server/bare/utils/tts-stats.js +10 -0
- package/dist/server/rpc/create-server.d.ts +4 -1
- package/dist/server/rpc/create-server.js +5 -1
- package/dist/server/rpc/delegate-transport.js +36 -29
- package/dist/server/rpc/handle-request.js +28 -3
- package/dist/server/rpc/handler-registry.js +7 -1
- package/dist/server/rpc/handler-utils.d.ts +5 -0
- package/dist/server/rpc/handler-utils.js +22 -0
- package/dist/server/rpc/handlers/cancel-delegated.js +3 -5
- package/dist/server/rpc/handlers/cancelHandler.js +4 -5
- package/dist/server/rpc/handlers/completion-stream-delegated.js +14 -5
- package/dist/server/rpc/handlers/delete-cache.js +8 -3
- package/dist/server/rpc/handlers/download-asset.js +3 -24
- package/dist/server/rpc/handlers/get-loaded-model-info.d.ts +3 -0
- package/dist/server/rpc/handlers/get-loaded-model-info.js +39 -0
- package/dist/server/rpc/handlers/get-model-info.js +93 -84
- package/dist/server/rpc/handlers/heartbeat-delegated.js +2 -2
- package/dist/server/rpc/handlers/index.d.ts +6 -0
- package/dist/server/rpc/handlers/index.js +6 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +35 -9
- package/dist/server/rpc/handlers/load-model/download-manager.js +142 -38
- package/dist/server/rpc/handlers/load-model/download-stats.d.ts +5 -4
- package/dist/server/rpc/handlers/load-model/download-stats.js +49 -8
- package/dist/server/rpc/handlers/load-model/handler.js +31 -50
- package/dist/server/rpc/handlers/load-model/http.d.ts +2 -2
- package/dist/server/rpc/handlers/load-model/http.js +97 -142
- package/dist/server/rpc/handlers/load-model/hyperdrive.d.ts +2 -2
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +55 -74
- package/dist/server/rpc/handlers/load-model/index.d.ts +0 -1
- package/dist/server/rpc/handlers/load-model/index.js +0 -1
- package/dist/server/rpc/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/server/rpc/handlers/load-model/registry-client-options.js +19 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.js +208 -0
- package/dist/server/rpc/handlers/load-model/registry-download-utils.d.ts +24 -0
- package/dist/server/rpc/handlers/load-model/registry-download-utils.js +114 -0
- package/dist/server/rpc/handlers/load-model/registry.d.ts +2 -20
- package/dist/server/rpc/handlers/load-model/registry.js +109 -381
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +15 -0
- package/dist/server/rpc/handlers/load-model/resolve-session.js +61 -0
- package/dist/server/rpc/handlers/load-model/resolve.d.ts +3 -3
- package/dist/server/rpc/handlers/load-model/resolve.js +14 -23
- package/dist/server/rpc/handlers/load-model/types.d.ts +11 -4
- package/dist/server/rpc/handlers/load-model/types.js +28 -1
- package/dist/server/rpc/handlers/load-model-delegated.js +2 -4
- package/dist/server/rpc/handlers/plugin-dispatch.js +9 -5
- package/dist/server/rpc/handlers/plugin-invoke.js +1 -1
- package/dist/server/rpc/handlers/provideHandler/connection.js +13 -2
- package/dist/server/rpc/handlers/provideHandler/handler.js +42 -29
- package/dist/server/rpc/handlers/state.d.ts +3 -0
- package/dist/server/rpc/handlers/state.js +7 -0
- package/dist/server/rpc/handlers/stopProvideHandler.d.ts +2 -2
- package/dist/server/rpc/handlers/stopProvideHandler.js +12 -7
- package/dist/server/rpc/handlers/text-to-speech-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/text-to-speech-stream.js +4 -0
- package/dist/server/rpc/handlers/unload-model-delegated.js +3 -3
- package/dist/server/rpc/profiling/operation-metrics.js +20 -0
- package/dist/server/rpc/rpc-utils.js +1 -1
- package/dist/server/rpc/transport-selector.d.ts +3 -0
- package/dist/server/rpc/transport-selector.js +18 -0
- package/dist/server/utils/cache/auto-history.d.ts +4 -0
- package/dist/server/utils/cache/auto-history.js +22 -0
- package/dist/server/utils/{cache.d.ts → cache/paths.d.ts} +22 -5
- package/dist/server/utils/{cache.js → cache/paths.js} +36 -20
- package/dist/server/utils/cache/types.d.ts +8 -0
- package/dist/server/utils/cache/types.js +1 -0
- package/dist/server/utils/checksum.d.ts +2 -2
- package/dist/server/utils/checksum.js +1 -1
- package/dist/server/utils/clear-storage.d.ts +5 -0
- package/dist/server/utils/clear-storage.js +17 -0
- package/dist/server/utils/completion-normalizer.d.ts +17 -0
- package/dist/server/utils/completion-normalizer.js +301 -0
- package/dist/server/utils/expand-gguf-shards.d.ts +3 -0
- package/dist/server/utils/expand-gguf-shards.js +22 -0
- package/dist/server/utils/formatting.js +1 -1
- package/dist/server/utils/index.d.ts +4 -2
- package/dist/server/utils/index.js +4 -2
- package/dist/server/utils/qvac-paths.d.ts +2 -0
- package/dist/server/utils/qvac-paths.js +5 -0
- package/dist/server/utils/response-format.d.ts +3 -0
- package/dist/server/utils/response-format.js +20 -0
- package/dist/server/utils/tool-integration.d.ts +15 -4
- package/dist/server/utils/tool-integration.js +21 -41
- package/dist/server/utils/tools/dialect.d.ts +8 -0
- package/dist/server/utils/tools/dialect.js +12 -0
- package/dist/server/utils/tools/index.d.ts +3 -0
- package/dist/server/utils/tools/index.js +2 -0
- package/dist/server/utils/tools/parser.d.ts +6 -0
- package/dist/server/utils/tools/parser.js +40 -0
- package/dist/server/utils/tools/parsers/hermes.d.ts +4 -0
- package/dist/server/utils/tools/parsers/hermes.js +100 -0
- package/dist/server/utils/tools/parsers/json.d.ts +6 -0
- package/dist/server/utils/tools/parsers/json.js +115 -0
- package/dist/server/utils/tools/parsers/pythonic.d.ts +4 -0
- package/dist/server/utils/tools/parsers/pythonic.js +267 -0
- package/dist/server/utils/tools/shared.d.ts +18 -0
- package/dist/server/utils/tools/shared.js +54 -0
- package/dist/server/utils/worker-lock.d.ts +3 -0
- package/dist/server/utils/worker-lock.js +67 -0
- package/dist/server/worker-core.d.ts +16 -1
- package/dist/server/worker-core.js +114 -14
- package/dist/utils/aggregate-events.d.ts +16 -0
- package/dist/utils/aggregate-events.js +52 -0
- package/dist/utils/cache-normalize.d.ts +10 -0
- package/dist/utils/cache-normalize.js +23 -0
- package/dist/utils/encoding.d.ts +8 -0
- package/dist/utils/encoding.js +13 -0
- package/dist/utils/errors-client.d.ts +12 -0
- package/dist/utils/errors-client.js +20 -0
- package/dist/utils/errors-server.d.ts +14 -0
- package/dist/utils/errors-server.js +31 -0
- package/dist/utils/load-model-validation.d.ts +7 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/expo/plugins/patches/android-link.mjs +1 -1
- package/expo/plugins/patches/ios-link.mjs +1 -1
- package/package.json +15 -19
- package/dist/examples/translation/translation-opus.d.ts +0 -2
- package/dist/examples/translation/translation-opus.js +0 -33
- package/dist/server/bare/utils/loader-adapter.d.ts +0 -7
- package/dist/server/bare/utils/loader-adapter.js +0 -9
- package/dist/server/utils/model-path.d.ts +0 -5
- package/dist/server/utils/model-path.js +0 -7
- package/dist/server/utils/tool-parser.d.ts +0 -7
- package/dist/server/utils/tool-parser.js +0 -248
|
@@ -61,14 +61,8 @@ try {
|
|
|
61
61
|
// Consume tool call events
|
|
62
62
|
const toolsTask = (async () => {
|
|
63
63
|
for await (const evt of result.toolCallStream) {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
console.log(` ID: ${evt.call.id}`);
|
|
67
|
-
}
|
|
68
|
-
else if (evt.type === "toolCallError") {
|
|
69
|
-
console.warn(`\n⚠️ Tool Error: ${evt.error.message}`);
|
|
70
|
-
console.warn(` Code: ${evt.error.code}`);
|
|
71
|
-
}
|
|
64
|
+
console.log(`\n\n→ Tool Call Detected: ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
|
|
65
|
+
console.log(` ID: ${evt.call.id}`);
|
|
72
66
|
}
|
|
73
67
|
})();
|
|
74
68
|
await Promise.all([tokensTask, toolsTask]);
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { loadModel, completion, unloadModel, QWEN3_600M_INST_Q4, } from "@qvac/sdk";
|
|
2
|
+
// Demonstrates per-request structured output via `responseFormat`.
|
|
3
|
+
// Three modes are exercised against the same model so you can eyeball the
|
|
4
|
+
// difference between free-form text, free-form JSON, and a strict JSON Schema.
|
|
5
|
+
//
|
|
6
|
+
// Usage:
|
|
7
|
+
// bun run examples/llamacpp-structured-output.ts
|
|
8
|
+
const PERSON_SCHEMA = {
|
|
9
|
+
type: "object",
|
|
10
|
+
properties: {
|
|
11
|
+
name: { type: "string" },
|
|
12
|
+
age: { type: "integer" },
|
|
13
|
+
occupation: { type: "string" },
|
|
14
|
+
},
|
|
15
|
+
required: ["name", "age", "occupation"],
|
|
16
|
+
additionalProperties: false,
|
|
17
|
+
};
|
|
18
|
+
const HISTORY = [
|
|
19
|
+
{
|
|
20
|
+
role: "system",
|
|
21
|
+
content: "You extract structured information about people from short bios. /no_think",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
role: "user",
|
|
25
|
+
content: "Hi, I'm Alice, I'm 30 years old and I work as a data engineer.",
|
|
26
|
+
},
|
|
27
|
+
];
|
|
28
|
+
// `json_object` mode only enforces that the output is *some* valid JSON object
|
|
29
|
+
// — it doesn't pin the keys. Small models (Qwen3-0.6B in this example) will
|
|
30
|
+
// often emit `{}` because that's the shortest valid completion under the
|
|
31
|
+
// grammar. The same gotcha is documented for OpenAI's `response_format:
|
|
32
|
+
// json_object`. To get useful keys you either need a stronger prompt + a
|
|
33
|
+
// larger model, or — better — switch to `json_schema` (mode 3 below) so the
|
|
34
|
+
// grammar itself forces the keys.
|
|
35
|
+
const JSON_OBJECT_HISTORY = [
|
|
36
|
+
{
|
|
37
|
+
role: "system",
|
|
38
|
+
content: 'You extract structured information about people from short bios and reply ONLY with a JSON object containing "name", "age", and "occupation". /no_think',
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
role: "user",
|
|
42
|
+
content: "Hi, I'm Alice, I'm 30 years old and I work as a data engineer.",
|
|
43
|
+
},
|
|
44
|
+
];
|
|
45
|
+
// Drains the run via the canonical `events` / `final` surface and returns the
|
|
46
|
+
// content text. `tokenStream` / `text` / `stats` still work but are legacy and
|
|
47
|
+
// will be deprecated, so new code should consume `events` for streaming and
|
|
48
|
+
// await `final.contentText` for the aggregated result.
|
|
49
|
+
async function runToContent(run) {
|
|
50
|
+
for await (const event of run.events) {
|
|
51
|
+
if (event.type === "contentDelta")
|
|
52
|
+
process.stdout.write(event.text);
|
|
53
|
+
}
|
|
54
|
+
const final = await run.final;
|
|
55
|
+
process.stdout.write("\n");
|
|
56
|
+
return final.contentText;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
const modelId = await loadModel({
|
|
60
|
+
modelSrc: QWEN3_600M_INST_Q4,
|
|
61
|
+
modelType: "llm",
|
|
62
|
+
onProgress: (p) => process.stdout.write(`\rLoading: ${p.percentage.toFixed(1)}%`),
|
|
63
|
+
});
|
|
64
|
+
console.log(`\nModel loaded: ${modelId}\n`);
|
|
65
|
+
console.log("--- 1. responseFormat: text (baseline, free-form) ---");
|
|
66
|
+
await runToContent(completion({
|
|
67
|
+
modelId,
|
|
68
|
+
history: HISTORY,
|
|
69
|
+
stream: true,
|
|
70
|
+
responseFormat: { type: "text" },
|
|
71
|
+
}));
|
|
72
|
+
console.log("\n--- 2. responseFormat: json_object (any valid JSON) ---");
|
|
73
|
+
const jsonObjectOut = await runToContent(completion({
|
|
74
|
+
modelId,
|
|
75
|
+
history: JSON_OBJECT_HISTORY,
|
|
76
|
+
stream: true,
|
|
77
|
+
responseFormat: { type: "json_object" },
|
|
78
|
+
}));
|
|
79
|
+
console.log("parsed:", JSON.parse(jsonObjectOut.trim()));
|
|
80
|
+
console.log("\n--- 3. responseFormat: json_schema (strict shape) ---");
|
|
81
|
+
const jsonSchemaOut = await runToContent(completion({
|
|
82
|
+
modelId,
|
|
83
|
+
history: HISTORY,
|
|
84
|
+
stream: true,
|
|
85
|
+
responseFormat: {
|
|
86
|
+
type: "json_schema",
|
|
87
|
+
json_schema: {
|
|
88
|
+
name: "person",
|
|
89
|
+
schema: PERSON_SCHEMA,
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
}));
|
|
93
|
+
// Output is guaranteed schema-valid JSON: object with exactly
|
|
94
|
+
// {name: string, age: integer, occupation: string}, no extras.
|
|
95
|
+
const parsed = JSON.parse(jsonSchemaOut.trim());
|
|
96
|
+
console.log("parsed:", parsed);
|
|
97
|
+
console.log("schema-valid:", typeof parsed.name === "string" &&
|
|
98
|
+
Number.isInteger(parsed.age) &&
|
|
99
|
+
typeof parsed.occupation === "string" &&
|
|
100
|
+
Object.keys(parsed).sort().join(",") === "age,name,occupation");
|
|
101
|
+
await unloadModel({ modelId });
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
console.error("Error:", error);
|
|
105
|
+
process.exit(1);
|
|
106
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { completion, loadModel, unloadModel, } from "@qvac/sdk";
|
|
3
|
+
// LFM2-1.2B-Tool emits Pythonic-style calls: [get_weather(city="Tokyo")].
|
|
4
|
+
const LFM_TOOL_HF = "https://huggingface.co/LiquidAI/LFM2-1.2B-Tool-GGUF/resolve/main/LFM2-1.2B-Tool-Q4_K_M.gguf";
|
|
5
|
+
const weatherSchema = z.object({
|
|
6
|
+
city: z.string().describe("City name"),
|
|
7
|
+
});
|
|
8
|
+
const horoscopeSchema = z.object({
|
|
9
|
+
sign: z.string().describe("An astrological sign like Taurus or Aquarius"),
|
|
10
|
+
});
|
|
11
|
+
const tools = [
|
|
12
|
+
{
|
|
13
|
+
name: "get_weather",
|
|
14
|
+
description: "Get current weather for a city",
|
|
15
|
+
parameters: weatherSchema,
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
name: "get_horoscope",
|
|
19
|
+
description: "Get today's horoscope for an astrological sign",
|
|
20
|
+
parameters: horoscopeSchema,
|
|
21
|
+
},
|
|
22
|
+
];
|
|
23
|
+
try {
|
|
24
|
+
const modelId = await loadModel({
|
|
25
|
+
modelSrc: LFM_TOOL_HF,
|
|
26
|
+
modelType: "llm",
|
|
27
|
+
modelConfig: { ctx_size: 4096, tools: true },
|
|
28
|
+
onProgress: (progress) => console.log(`Loading: ${progress.percentage.toFixed(1)}%`),
|
|
29
|
+
});
|
|
30
|
+
console.log(`✅ Model loaded: ${modelId}`);
|
|
31
|
+
const history = [
|
|
32
|
+
{
|
|
33
|
+
role: "system",
|
|
34
|
+
content: "You are a helpful assistant that can call tools to look up weather and horoscopes.",
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
role: "user",
|
|
38
|
+
content: "What's the weather in Tokyo and my horoscope for Aquarius?",
|
|
39
|
+
},
|
|
40
|
+
];
|
|
41
|
+
console.log("\n🤖 Streaming...\n");
|
|
42
|
+
const result = completion({ modelId, history, stream: true, tools });
|
|
43
|
+
const tokensTask = (async () => {
|
|
44
|
+
for await (const token of result.tokenStream) {
|
|
45
|
+
process.stdout.write(token);
|
|
46
|
+
}
|
|
47
|
+
})();
|
|
48
|
+
const toolsTask = (async () => {
|
|
49
|
+
for await (const evt of result.toolCallStream) {
|
|
50
|
+
console.log(`\n→ ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
|
|
51
|
+
}
|
|
52
|
+
})();
|
|
53
|
+
await Promise.all([tokensTask, toolsTask]);
|
|
54
|
+
const toolCalls = await result.toolCalls;
|
|
55
|
+
console.log("\n\n📋 Final tool calls:");
|
|
56
|
+
if (toolCalls.length > 0) {
|
|
57
|
+
for (const call of toolCalls) {
|
|
58
|
+
console.log(` - ${call.name}(${JSON.stringify(call.arguments)})`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
console.log(" (none)");
|
|
63
|
+
}
|
|
64
|
+
await unloadModel({ modelId });
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
console.error("❌ Error:", error);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|
package/dist/examples/plugins.js
CHANGED
|
@@ -46,7 +46,7 @@ process.on("exit", cleanup);
|
|
|
46
46
|
// Point config before importing the SDK.
|
|
47
47
|
// In a real app, place qvac.config.json (or .js/.ts) in your project root (auto-discovered).
|
|
48
48
|
process.env["QVAC_CONFIG_PATH"] = configPath;
|
|
49
|
-
const { completion, translate, embed, loadModel, unloadModel, LLAMA_3_2_1B_INST_Q4_0,
|
|
49
|
+
const { completion, translate, embed, loadModel, unloadModel, LLAMA_3_2_1B_INST_Q4_0, BERGAMOT_EN_ES, GTE_LARGE_FP16, } = await import("@qvac/sdk");
|
|
50
50
|
console.log("1. LLM Completion (llamacpp-completion plugin)");
|
|
51
51
|
try {
|
|
52
52
|
const llmModelId = await loadModel({
|
|
@@ -78,12 +78,12 @@ catch (error) {
|
|
|
78
78
|
console.log("2. Translation (nmtcpp-translation plugin)");
|
|
79
79
|
try {
|
|
80
80
|
const nmtModelId = await loadModel({
|
|
81
|
-
modelSrc:
|
|
81
|
+
modelSrc: BERGAMOT_EN_ES,
|
|
82
82
|
modelType: "nmt",
|
|
83
83
|
modelConfig: {
|
|
84
|
-
engine: "
|
|
84
|
+
engine: "Bergamot",
|
|
85
85
|
from: "en",
|
|
86
|
-
to: "
|
|
86
|
+
to: "es",
|
|
87
87
|
},
|
|
88
88
|
onProgress: (p) => console.log(` Loading NMT: ${p.percentage.toFixed(1)}%`),
|
|
89
89
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadModel, LLAMA_3_2_1B_INST_Q4_0, completion, unloadModel, suspend, resume, } from "@qvac/sdk";
|
|
1
|
+
import { loadModel, LLAMA_3_2_1B_INST_Q4_0, completion, unloadModel, suspend, resume, state, } from "@qvac/sdk";
|
|
2
2
|
try {
|
|
3
3
|
// Load a model
|
|
4
4
|
const modelId = await loadModel({
|
|
@@ -9,6 +9,7 @@ try {
|
|
|
9
9
|
},
|
|
10
10
|
});
|
|
11
11
|
console.log("✅ Model loaded\n");
|
|
12
|
+
console.log(`📊 Lifecycle state: ${await state()}\n`);
|
|
12
13
|
// Run a completion before suspending
|
|
13
14
|
console.log("--- Completion before suspend ---");
|
|
14
15
|
const result1 = completion({
|
|
@@ -23,14 +24,30 @@ try {
|
|
|
23
24
|
// Suspend all networking and storage (e.g. app going to background)
|
|
24
25
|
console.log("⏸️ Suspending...");
|
|
25
26
|
await suspend();
|
|
26
|
-
console.log(
|
|
27
|
+
console.log(`📊 Lifecycle state: ${await state()}\n`);
|
|
28
|
+
try {
|
|
29
|
+
await completion({
|
|
30
|
+
modelId,
|
|
31
|
+
history: [{ role: "user", content: "This should fail" }],
|
|
32
|
+
stream: false,
|
|
33
|
+
}).text;
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
const name = error.name;
|
|
37
|
+
if (name === "LIFECYCLE_OPERATION_BLOCKED") {
|
|
38
|
+
console.log(`🚫 Operation blocked while suspended (${name})`);
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
27
44
|
// Simulate time in background
|
|
28
|
-
console.log("💤 Simulating 3 seconds in background...");
|
|
45
|
+
console.log("\n💤 Simulating 3 seconds in background...");
|
|
29
46
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
30
47
|
// Resume when returning to foreground
|
|
31
48
|
console.log("▶️ Resuming...");
|
|
32
49
|
await resume();
|
|
33
|
-
console.log(
|
|
50
|
+
console.log(`📊 Lifecycle state: ${await state()}\n`);
|
|
34
51
|
// Run another completion after resuming
|
|
35
52
|
console.log("--- Completion after resume ---");
|
|
36
53
|
const result2 = completion({
|
|
@@ -9,29 +9,12 @@
|
|
|
9
9
|
* Requirements: FFmpeg installed, microphone access.
|
|
10
10
|
*/
|
|
11
11
|
import { loadModel, unloadModel, transcribe, PARAKEET_TDT_ENCODER_FP32, PARAKEET_TDT_DECODER_FP32, PARAKEET_TDT_VOCAB, PARAKEET_TDT_PREPROCESSOR_FP32, } from "@qvac/sdk";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { spawnSync } from "child_process";
|
|
13
|
+
import { startMicrophone } from "../audio/mic-input";
|
|
14
14
|
const SAMPLE_RATE = 16000;
|
|
15
15
|
const BYTES_PER_SAMPLE = 2; // s16le
|
|
16
16
|
const CHUNK_DURATION_S = 3;
|
|
17
17
|
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHUNK_DURATION_S;
|
|
18
|
-
function getAudioInputArgs() {
|
|
19
|
-
switch (platform()) {
|
|
20
|
-
case "darwin":
|
|
21
|
-
return ["-f", "avfoundation", "-i", ":0"];
|
|
22
|
-
case "win32":
|
|
23
|
-
return [
|
|
24
|
-
"-f",
|
|
25
|
-
"dshow",
|
|
26
|
-
"-i",
|
|
27
|
-
"audio=@device_cm_{33D9A762-90C8-11D0-BD43-00A0C911CE86}\\wave_{58C07110-A4FD-4FF8-BA10-5A3C14389F71}",
|
|
28
|
-
];
|
|
29
|
-
case "linux":
|
|
30
|
-
return ["-f", "pulse", "-i", "default"];
|
|
31
|
-
default:
|
|
32
|
-
throw new Error(`Unsupported platform: ${platform()}`);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
18
|
// ── Main ──
|
|
36
19
|
try {
|
|
37
20
|
const r = spawnSync("ffmpeg", ["-version"], { stdio: "ignore" });
|
|
@@ -55,20 +38,10 @@ const modelId = await loadModel({
|
|
|
55
38
|
onProgress: (p) => console.log(`Download: ${p.percentage.toFixed(1)}%`),
|
|
56
39
|
});
|
|
57
40
|
console.log("Model loaded.\n");
|
|
58
|
-
const ffmpeg =
|
|
59
|
-
|
|
60
|
-
"
|
|
61
|
-
|
|
62
|
-
"-ac",
|
|
63
|
-
"1",
|
|
64
|
-
"-sample_fmt",
|
|
65
|
-
"s16",
|
|
66
|
-
"-f",
|
|
67
|
-
"s16le",
|
|
68
|
-
"pipe:1",
|
|
69
|
-
], { stdio: ["ignore", "pipe", "ignore"] });
|
|
70
|
-
if (!ffmpeg.stdout)
|
|
71
|
-
throw new Error("Failed to open microphone");
|
|
41
|
+
const ffmpeg = startMicrophone({
|
|
42
|
+
sampleRate: SAMPLE_RATE,
|
|
43
|
+
format: "s16le",
|
|
44
|
+
});
|
|
72
45
|
let buffer = Buffer.alloc(0);
|
|
73
46
|
let processing = false;
|
|
74
47
|
console.log("Listening... speak and pause to see transcriptions.\n");
|
|
@@ -51,7 +51,7 @@ try {
|
|
|
51
51
|
});
|
|
52
52
|
console.log(`Model loaded: ${modelId}\n`);
|
|
53
53
|
console.log("Opening live session...");
|
|
54
|
-
const session = await transcribeStream({ modelId });
|
|
54
|
+
const session = await transcribeStream({ modelId, metadata: true });
|
|
55
55
|
console.log("Session open. Streaming audio...\n");
|
|
56
56
|
const ffmpeg = spawn("ffmpeg", [
|
|
57
57
|
"-i", SAMPLE_FILE,
|
|
@@ -76,14 +76,16 @@ try {
|
|
|
76
76
|
session.end();
|
|
77
77
|
});
|
|
78
78
|
const segments = [];
|
|
79
|
-
for await (const
|
|
80
|
-
segments.push(
|
|
81
|
-
|
|
79
|
+
for await (const segment of session) {
|
|
80
|
+
segments.push(segment);
|
|
81
|
+
const start = (segment.startMs / 1000).toFixed(2);
|
|
82
|
+
const end = (segment.endMs / 1000).toFixed(2);
|
|
83
|
+
console.log(` [${segments.length}] [${start}s → ${end}s] (id=${segment.id}, append=${segment.append}) ${segment.text.trim()}`);
|
|
82
84
|
}
|
|
83
85
|
console.log("\n=== Results ===");
|
|
84
86
|
console.log(`Segments: ${segments.length}`);
|
|
85
87
|
if (segments.length > 0) {
|
|
86
|
-
console.log(`Transcript: ${segments.join(" ")}`);
|
|
88
|
+
console.log(`Transcript: ${segments.map((s) => s.text.trim()).join(" ")}`);
|
|
87
89
|
}
|
|
88
90
|
else {
|
|
89
91
|
console.log("WARNING: No transcription segments received!");
|
|
@@ -53,11 +53,20 @@ try {
|
|
|
53
53
|
},
|
|
54
54
|
});
|
|
55
55
|
console.log(`✅ Whisper model loaded with ID: ${modelId}`);
|
|
56
|
-
// Perform transcription
|
|
56
|
+
// Perform transcription with per-segment metadata
|
|
57
57
|
console.log("🎧 Transcribing audio...");
|
|
58
|
-
const
|
|
58
|
+
const segments = await transcribe({
|
|
59
|
+
modelId,
|
|
60
|
+
audioChunk: audioFilePath,
|
|
61
|
+
metadata: true,
|
|
62
|
+
});
|
|
59
63
|
console.log("📝 Transcription result:");
|
|
60
|
-
|
|
64
|
+
for (const segment of segments) {
|
|
65
|
+
const start = (segment.startMs / 1000).toFixed(2);
|
|
66
|
+
const end = (segment.endMs / 1000).toFixed(2);
|
|
67
|
+
console.log(` [${start}s → ${end}s] (id=${segment.id}, append=${segment.append}) ${segment.text}`);
|
|
68
|
+
}
|
|
69
|
+
console.log(`\nFull transcript: ${segments.map((s) => s.text).join("").trim()}`);
|
|
61
70
|
// Unload the model when done
|
|
62
71
|
console.log("🧹 Unloading Whisper model...");
|
|
63
72
|
await unloadModel({ modelId });
|
|
@@ -9,26 +9,9 @@
|
|
|
9
9
|
* Requirements: FFmpeg installed, microphone access.
|
|
10
10
|
*/
|
|
11
11
|
import { loadModel, unloadModel, transcribeStream, WHISPER_TINY, VAD_SILERO_5_1_2, } from "@qvac/sdk";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { spawnSync } from "child_process";
|
|
13
|
+
import { startMicrophone } from "../audio/mic-input";
|
|
14
14
|
const SAMPLE_RATE = 16000;
|
|
15
|
-
function getAudioInputArgs() {
|
|
16
|
-
switch (platform()) {
|
|
17
|
-
case "darwin":
|
|
18
|
-
return ["-f", "avfoundation", "-i", ":0"];
|
|
19
|
-
case "win32":
|
|
20
|
-
return [
|
|
21
|
-
"-f",
|
|
22
|
-
"dshow",
|
|
23
|
-
"-i",
|
|
24
|
-
"audio=@device_cm_{33D9A762-90C8-11D0-BD43-00A0C911CE86}\\wave_{58C07110-A4FD-4FF8-BA10-5A3C14389F71}",
|
|
25
|
-
];
|
|
26
|
-
case "linux":
|
|
27
|
-
return ["-f", "pulse", "-i", "default"];
|
|
28
|
-
default:
|
|
29
|
-
throw new Error(`Unsupported platform: ${platform()}`);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
15
|
// ── Main ──
|
|
33
16
|
try {
|
|
34
17
|
const r = spawnSync("ffmpeg", ["-version"], { stdio: "ignore" });
|
|
@@ -63,16 +46,10 @@ const modelId = await loadModel({
|
|
|
63
46
|
},
|
|
64
47
|
});
|
|
65
48
|
console.log("Model loaded.\n");
|
|
66
|
-
const ffmpeg =
|
|
67
|
-
|
|
68
|
-
"
|
|
69
|
-
|
|
70
|
-
"-sample_fmt", "flt",
|
|
71
|
-
"-f", "f32le",
|
|
72
|
-
"pipe:1",
|
|
73
|
-
], { stdio: ["ignore", "pipe", "ignore"] });
|
|
74
|
-
if (!ffmpeg.stdout)
|
|
75
|
-
throw new Error("Failed to open microphone");
|
|
49
|
+
const ffmpeg = startMicrophone({
|
|
50
|
+
sampleRate: SAMPLE_RATE,
|
|
51
|
+
format: "f32le",
|
|
52
|
+
});
|
|
76
53
|
const session = await transcribeStream({ modelId });
|
|
77
54
|
ffmpeg.stdout.on("data", (chunk) => session.write(chunk));
|
|
78
55
|
console.log("Listening... speak and pause to see transcriptions.\n");
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { loadModel, translate, unloadModel,
|
|
1
|
+
import { loadModel, translate, unloadModel, BERGAMOT_EN_ES, } from "@qvac/sdk";
|
|
2
2
|
try {
|
|
3
3
|
const modelId = await loadModel({
|
|
4
|
-
modelSrc:
|
|
4
|
+
modelSrc: BERGAMOT_EN_ES,
|
|
5
5
|
modelType: "nmt",
|
|
6
6
|
modelConfig: {
|
|
7
|
-
engine: "
|
|
7
|
+
engine: "Bergamot",
|
|
8
8
|
from: "en",
|
|
9
|
-
to: "
|
|
9
|
+
to: "es",
|
|
10
10
|
},
|
|
11
11
|
});
|
|
12
12
|
console.log(`✅ Model loaded: ${modelId}`);
|
|
@@ -18,7 +18,7 @@ try {
|
|
|
18
18
|
modelType: "nmt",
|
|
19
19
|
stream: true,
|
|
20
20
|
});
|
|
21
|
-
process.stdout.write("Translated text EN ->
|
|
21
|
+
process.stdout.write("Translated text EN -> ES: ");
|
|
22
22
|
for await (const token of streamResult.tokenStream) {
|
|
23
23
|
process.stdout.write(token);
|
|
24
24
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stream an LLM and forward token deltas through the duplex TTS path (`textToSpeechStream` →
|
|
3
|
+
* `runStreaming` with `accumulateSentences: true`). Text is filtered before TTS / console: any
|
|
4
|
+
* `<...>` span (including across tokens) is removed, and `*` characters are stripped.
|
|
5
|
+
*
|
|
6
|
+
* Each synthesized phrase is played as soon as PCM arrives (`playPcmInt16Chunk`). A combined WAV
|
|
7
|
+
* is written at the end for inspection.
|
|
8
|
+
*
|
|
9
|
+
* Prerequisites: Bun, QVAC worker, registry access, macOS `afplay` / Linux `aplay` (or Windows
|
|
10
|
+
* PowerShell player) for chunk playback.
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* bun run examples/tts/llm-to-tts-streaming.ts
|
|
14
|
+
*/
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=llm-to-tts-streaming.d.ts.map
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stream an LLM and forward token deltas through the duplex TTS path (`textToSpeechStream` →
|
|
3
|
+
* `runStreaming` with `accumulateSentences: true`). Text is filtered before TTS / console: any
|
|
4
|
+
* `<...>` span (including across tokens) is removed, and `*` characters are stripped.
|
|
5
|
+
*
|
|
6
|
+
* Each synthesized phrase is played as soon as PCM arrives (`playPcmInt16Chunk`). A combined WAV
|
|
7
|
+
* is written at the end for inspection.
|
|
8
|
+
*
|
|
9
|
+
* Prerequisites: Bun, QVAC worker, registry access, macOS `afplay` / Linux `aplay` (or Windows
|
|
10
|
+
* PowerShell player) for chunk playback.
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* bun run examples/tts/llm-to-tts-streaming.ts
|
|
14
|
+
*/
|
|
15
|
+
import { completion, loadModel, textToSpeechStream, unloadModel, LLAMA_3_2_1B_INST_Q4_0, TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32, TTS_SUPERTONIC2_OFFICIAL_DURATION_PREDICTOR_SUPERTONE_FP32, TTS_SUPERTONIC2_OFFICIAL_VECTOR_ESTIMATOR_SUPERTONE_FP32, TTS_SUPERTONIC2_OFFICIAL_VOCODER_SUPERTONE_FP32, TTS_SUPERTONIC2_OFFICIAL_UNICODE_INDEXER_SUPERTONE_FP32, TTS_SUPERTONIC2_OFFICIAL_TTS_CONFIG_SUPERTONE, TTS_SUPERTONIC2_OFFICIAL_VOICE_STYLE_SUPERTONE, } from "@qvac/sdk";
|
|
16
|
+
import { createWav, playPcmInt16Chunk } from "./utils.js";
|
|
17
|
+
const SUPERTONIC_SAMPLE_RATE = 44100;
|
|
18
|
+
/**
|
|
19
|
+
* Drop angle-bracket spans (e.g. `<think>…</think>`, `<|foo|>`) even when split across tokens,
|
|
20
|
+
* and strip `*` (markdown emphasis) from text passed to TTS and the console.
|
|
21
|
+
*/
|
|
22
|
+
function createAngleBracketAndStarFilter() {
|
|
23
|
+
let inAngle = false;
|
|
24
|
+
return function filterChunk(chunk) {
|
|
25
|
+
let out = "";
|
|
26
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
27
|
+
const c = chunk[i];
|
|
28
|
+
if (inAngle) {
|
|
29
|
+
if (c === ">") {
|
|
30
|
+
inAngle = false;
|
|
31
|
+
}
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (c === "<") {
|
|
35
|
+
inAngle = true;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (c === "*") {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
out += c;
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Append PCM without `push(...huge)` — a spread of hundreds of thousands of
|
|
48
|
+
* arguments blows the V8 / Bare argument-count limit. Batching with
|
|
49
|
+
* `Array.prototype.push.apply` is both faster than `push(...slice)` and
|
|
50
|
+
* avoids allocating an intermediate rest array on the stack per call.
|
|
51
|
+
*/
|
|
52
|
+
function appendPcmSamples(target, chunk) {
|
|
53
|
+
const batch = 8192;
|
|
54
|
+
for (let i = 0; i < chunk.length; i += batch) {
|
|
55
|
+
const end = Math.min(i + batch, chunk.length);
|
|
56
|
+
Array.prototype.push.apply(target, chunk.slice(i, end));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
console.log(`Loading LLM from registry: ${LLAMA_3_2_1B_INST_Q4_0.name}`);
|
|
61
|
+
const llmModelId = await loadModel({
|
|
62
|
+
modelSrc: LLAMA_3_2_1B_INST_Q4_0.src,
|
|
63
|
+
modelType: "llm",
|
|
64
|
+
modelConfig: {
|
|
65
|
+
ctx_size: 2048,
|
|
66
|
+
},
|
|
67
|
+
onProgress: (progress) => console.log(`LLM load: ${progress.percentage.toFixed(1)}%`),
|
|
68
|
+
});
|
|
69
|
+
console.log(`LLM ready: ${llmModelId}`);
|
|
70
|
+
console.log("Loading Supertonic TTS (registry)…");
|
|
71
|
+
const ttsModelId = await loadModel({
|
|
72
|
+
modelSrc: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32.src,
|
|
73
|
+
modelType: "tts",
|
|
74
|
+
modelConfig: {
|
|
75
|
+
ttsEngine: "supertonic",
|
|
76
|
+
language: "en",
|
|
77
|
+
ttsSpeed: 1.05,
|
|
78
|
+
ttsNumInferenceSteps: 10,
|
|
79
|
+
ttsSupertonicMultilingual: true,
|
|
80
|
+
ttsTextEncoderSrc: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32.src,
|
|
81
|
+
ttsDurationPredictorSrc: TTS_SUPERTONIC2_OFFICIAL_DURATION_PREDICTOR_SUPERTONE_FP32.src,
|
|
82
|
+
ttsVectorEstimatorSrc: TTS_SUPERTONIC2_OFFICIAL_VECTOR_ESTIMATOR_SUPERTONE_FP32.src,
|
|
83
|
+
ttsVocoderSrc: TTS_SUPERTONIC2_OFFICIAL_VOCODER_SUPERTONE_FP32.src,
|
|
84
|
+
ttsUnicodeIndexerSrc: TTS_SUPERTONIC2_OFFICIAL_UNICODE_INDEXER_SUPERTONE_FP32.src,
|
|
85
|
+
ttsTtsConfigSrc: TTS_SUPERTONIC2_OFFICIAL_TTS_CONFIG_SUPERTONE.src,
|
|
86
|
+
ttsVoiceStyleSrc: TTS_SUPERTONIC2_OFFICIAL_VOICE_STYLE_SUPERTONE.src,
|
|
87
|
+
},
|
|
88
|
+
onProgress: (progress) => console.log(`TTS load: ${progress.percentage.toFixed(1)}%`),
|
|
89
|
+
});
|
|
90
|
+
console.log(`TTS ready: ${ttsModelId}`);
|
|
91
|
+
const prompt = "What is a constellation?";
|
|
92
|
+
console.log(`\nUser: ${prompt}\nAssistant (streaming):`);
|
|
93
|
+
const result = completion({
|
|
94
|
+
modelId: llmModelId,
|
|
95
|
+
history: [{ role: "user", content: prompt }],
|
|
96
|
+
stream: true,
|
|
97
|
+
});
|
|
98
|
+
const combinedPcm = [];
|
|
99
|
+
const ttsSession = await textToSpeechStream({
|
|
100
|
+
modelId: ttsModelId,
|
|
101
|
+
inputType: "text",
|
|
102
|
+
accumulateSentences: true,
|
|
103
|
+
});
|
|
104
|
+
let phraseIndex = 0;
|
|
105
|
+
const filterChunk = createAngleBracketAndStarFilter();
|
|
106
|
+
const drainPcm = (async () => {
|
|
107
|
+
for await (const m of ttsSession) {
|
|
108
|
+
if (m.buffer.length > 0) {
|
|
109
|
+
appendPcmSamples(combinedPcm, m.buffer);
|
|
110
|
+
phraseIndex += 1;
|
|
111
|
+
const preview = typeof m.sentenceChunk === "string"
|
|
112
|
+
? m.sentenceChunk.replace(/\s+/g, " ").trim().slice(0, 72)
|
|
113
|
+
: "";
|
|
114
|
+
console.log(`\n[TTS phrase ${phraseIndex}] ${m.buffer.length} samples${preview ? ` — "${preview}${preview.length >= 72 ? "..." : ""}"` : ""}`);
|
|
115
|
+
await playPcmInt16Chunk(m.buffer, SUPERTONIC_SAMPLE_RATE);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
})();
|
|
119
|
+
for await (const token of result.tokenStream) {
|
|
120
|
+
const cleaned = filterChunk(token);
|
|
121
|
+
if (cleaned.length > 0) {
|
|
122
|
+
process.stdout.write(cleaned);
|
|
123
|
+
ttsSession.write(cleaned);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
ttsSession.end();
|
|
127
|
+
await drainPcm;
|
|
128
|
+
const stats = await result.stats;
|
|
129
|
+
if (stats) {
|
|
130
|
+
console.log("\nLLM stats:", stats);
|
|
131
|
+
}
|
|
132
|
+
const outWav = "llm-to-tts-streaming-output.wav";
|
|
133
|
+
console.log(`\nWriting ${combinedPcm.length} samples to ${outWav} (full utterance; phrases were already played above).`);
|
|
134
|
+
createWav(combinedPcm, SUPERTONIC_SAMPLE_RATE, outWav);
|
|
135
|
+
await unloadModel({ modelId: llmModelId, clearStorage: false });
|
|
136
|
+
await unloadModel({ modelId: ttsModelId, clearStorage: false });
|
|
137
|
+
console.log("Models unloaded.");
|
|
138
|
+
process.exit(0);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
console.error("❌ Error:", error);
|
|
142
|
+
process.exit(1);
|
|
143
|
+
}
|
|
@@ -10,9 +10,9 @@ try {
|
|
|
10
10
|
modelConfig: {
|
|
11
11
|
ttsEngine: "supertonic",
|
|
12
12
|
language: "es",
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
ttsSpeed: 1.05,
|
|
14
|
+
ttsNumInferenceSteps: 5,
|
|
15
|
+
ttsSupertonicMultilingual: true, //false for English quality.
|
|
16
16
|
ttsTextEncoderSrc: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32.src,
|
|
17
17
|
ttsDurationPredictorSrc: TTS_SUPERTONIC2_OFFICIAL_DURATION_PREDICTOR_SUPERTONE_FP32.src,
|
|
18
18
|
ttsVectorEstimatorSrc: TTS_SUPERTONIC2_OFFICIAL_VECTOR_ESTIMATOR_SUPERTONE_FP32.src,
|
|
@@ -10,9 +10,9 @@ try {
|
|
|
10
10
|
modelConfig: {
|
|
11
11
|
ttsEngine: "supertonic",
|
|
12
12
|
language: "en",
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
ttsSpeed: 1.05,
|
|
14
|
+
ttsNumInferenceSteps: 5,
|
|
15
|
+
ttsSupertonicMultilingual: false, //false for English quality.
|
|
16
16
|
ttsTextEncoderSrc: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32.src,
|
|
17
17
|
ttsDurationPredictorSrc: TTS_SUPERTONIC2_OFFICIAL_DURATION_PREDICTOR_SUPERTONE_FP32.src,
|
|
18
18
|
ttsVectorEstimatorSrc: TTS_SUPERTONIC2_OFFICIAL_VECTOR_ESTIMATOR_SUPERTONE_FP32.src,
|