@qvac/sdk 0.12.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +55 -1225
- package/README.md +12 -0
- package/dist/client/api/bci-transcribe.d.ts +57 -0
- package/dist/client/api/bci-transcribe.js +172 -0
- package/dist/client/api/get-model-info.d.ts +1 -1
- package/dist/client/api/index.d.ts +1 -0
- package/dist/client/api/index.js +1 -0
- package/dist/client/api/video.d.ts +26 -6
- package/dist/client/api/video.js +30 -7
- package/dist/client/api/vla.d.ts +11 -7
- package/dist/client/api/vla.js +11 -7
- package/dist/client/config-loader/resolve-config.bare.d.ts +1 -1
- package/dist/client/config-loader/resolve-config.bare.js +21 -35
- package/dist/client/rpc/bare-client.js +2 -2
- package/dist/commands/bundle/constants.js +2 -0
- package/dist/examples/bci/bci-filesystem-streaming.d.ts +2 -0
- package/dist/examples/bci/bci-filesystem-streaming.js +77 -0
- package/dist/examples/bci/bci-filesystem.d.ts +2 -0
- package/dist/examples/bci/bci-filesystem.js +63 -0
- package/dist/examples/cache-management.js +0 -1
- package/dist/examples/cancel-by-request-id.js +0 -1
- package/dist/examples/classification/classify-image.js +1 -1
- package/dist/examples/completion-events.js +6 -2
- package/dist/examples/completion-stop-reason.d.ts +25 -0
- package/dist/examples/completion-stop-reason.js +65 -0
- package/dist/examples/concurrent-completion-collision.d.ts +2 -0
- package/dist/examples/concurrent-completion-collision.js +98 -0
- package/dist/examples/config-reload.js +1 -2
- package/dist/examples/default-config-usage.js +0 -1
- package/dist/examples/delegated-inference/composite.js +0 -1
- package/dist/examples/delegated-inference/consumer-profiled.js +0 -1
- package/dist/examples/delegated-inference/consumer.js +0 -1
- package/dist/examples/diffusion-esrgan-upscale.js +1 -3
- package/dist/examples/diffusion-flux2-klein-img2img.js +1 -1
- package/dist/examples/diffusion-flux2-klein.js +2 -2
- package/dist/examples/diffusion-img2img.js +1 -1
- package/dist/examples/diffusion-img2vid.d.ts +2 -0
- package/dist/examples/diffusion-img2vid.js +78 -0
- package/dist/examples/diffusion-simple.js +1 -1
- package/dist/examples/diffusion-txt2img.js +8 -3
- package/dist/examples/diffusion-txt2vid.js +2 -3
- package/dist/examples/download-with-blind-relays.js +0 -1
- package/dist/examples/embed-multi-gpu.js +1 -2
- package/dist/examples/embed-p2p.js +4 -2
- package/dist/examples/finetune/llamacpp-finetune.js +5 -3
- package/dist/examples/kv-cache-cleanup.js +0 -1
- package/dist/examples/kv-cache-custom-key.js +0 -1
- package/dist/examples/kv-cache-example.js +4 -2
- package/dist/examples/llamacpp-cache.js +0 -1
- package/dist/examples/llamacpp-dynamic-tools.js +4 -2
- package/dist/examples/llamacpp-filesystem.js +1 -1
- package/dist/examples/llamacpp-http.js +1 -1
- package/dist/examples/llamacpp-multi-gpu.js +2 -2
- package/dist/examples/llamacpp-multimodal.js +0 -1
- package/dist/examples/llamacpp-p2p.js +0 -1
- package/dist/examples/llamacpp-sharded.js +1 -1
- package/dist/examples/llamacpp-structured-output.js +0 -1
- package/dist/examples/logging-file-transport.js +0 -1
- package/dist/examples/logging-streaming.js +0 -2
- package/dist/examples/mcp-websearch.js +0 -1
- package/dist/examples/multi-model-demo.js +0 -2
- package/dist/examples/ocr-fasttext.js +0 -1
- package/dist/examples/plugins.js +1 -4
- package/dist/examples/profiling/basic.js +3 -2
- package/dist/examples/profiling/per-call.js +0 -1
- package/dist/examples/quickstart.js +0 -1
- package/dist/examples/rag/rag-chromadb.js +0 -1
- package/dist/examples/rag/rag-hyperdb/cancellation.js +0 -1
- package/dist/examples/rag/rag-hyperdb/ingest.js +0 -1
- package/dist/examples/rag/rag-hyperdb/pipeline.js +0 -1
- package/dist/examples/rag/rag-hyperdb/workspaces.js +0 -1
- package/dist/examples/rag/rag-lancedb.js +0 -1
- package/dist/examples/rag/rag-sqlite.js +0 -1
- package/dist/examples/suspend-resume.js +0 -1
- package/dist/examples/tools/llamacpp-native-tools.js +0 -1
- package/dist/examples/tools/llamacpp-tools-gemma4.js +1 -1
- package/dist/examples/tools/llamacpp-tools-pythonic.js +2 -2
- package/dist/examples/tools/llamacpp-tools-qwen35.js +1 -1
- package/dist/examples/transcription/parakeet-ctc-filesystem.js +1 -1
- package/dist/examples/transcription/parakeet-microphone-record.js +0 -1
- package/dist/examples/transcription/parakeet-microphone-stream.js +0 -1
- package/dist/examples/transcription/parakeet-sortformer.js +1 -2
- package/dist/examples/transcription/parakeet-tdt-filesystem.js +1 -1
- package/dist/examples/transcription/whispercpp-filesystem-streaming.js +10 -6
- package/dist/examples/transcription/whispercpp-filesystem.js +4 -2
- package/dist/examples/transcription/whispercpp-microphone-conversation.js +0 -1
- package/dist/examples/transcription/whispercpp-microphone-record.js +0 -1
- package/dist/examples/transcription/whispercpp-prompt.js +0 -1
- package/dist/examples/translation/translation-bergamot-batch.js +1 -2
- package/dist/examples/translation/translation-bergamot-pivot.js +3 -4
- package/dist/examples/translation/translation-bergamot.js +1 -2
- package/dist/examples/translation/translation-indic.js +1 -2
- package/dist/examples/translation/translation-llm-afriquegemma.js +3 -4
- package/dist/examples/translation/translation-llm-context.js +1 -2
- package/dist/examples/translation/translation-llm.js +2 -3
- package/dist/examples/translation/translation-stream.js +2 -3
- package/dist/examples/tts/chatterbox.js +1 -2
- package/dist/examples/tts/llm-to-tts-streaming.js +2 -4
- package/dist/examples/tts/supertonic-multilingual.js +1 -2
- package/dist/examples/tts/supertonic.js +1 -2
- package/dist/examples/vla-pi05.d.ts +2 -0
- package/dist/examples/vla-pi05.js +84 -0
- package/dist/examples/vla-smolvla.js +1 -1
- package/dist/examples/voice-assistant/voice-assistant.js +1 -4
- package/dist/index.d.ts +2 -2
- package/dist/index.js +9 -7
- package/dist/models/registry/models.d.ts +869 -186
- package/dist/models/registry/models.js +3160 -2453
- package/dist/models/update-models/companions.d.ts +7 -0
- package/dist/models/update-models/companions.js +56 -0
- package/dist/models/update-models/naming.js +10 -0
- package/dist/pear/pre.js +2 -0
- package/dist/schemas/bci-config.d.ts +36 -0
- package/dist/schemas/bci-config.js +56 -0
- package/dist/schemas/bci.d.ts +177 -0
- package/dist/schemas/bci.js +95 -0
- package/dist/schemas/common.d.ts +395 -51
- package/dist/schemas/common.js +5 -0
- package/dist/schemas/completion-event.d.ts +1 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/embed.d.ts +2 -0
- package/dist/schemas/embed.js +1 -0
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +4 -1
- package/dist/schemas/get-model-info.d.ts +2 -0
- package/dist/schemas/get-model-info.js +1 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +6 -2
- package/dist/schemas/load-model.d.ts +1841 -267
- package/dist/schemas/load-model.js +34 -1
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +19 -1
- package/dist/schemas/model-types.js +11 -0
- package/dist/schemas/ocr.d.ts +2 -0
- package/dist/schemas/plugin.d.ts +13 -5
- package/dist/schemas/plugin.js +13 -4
- package/dist/schemas/registry.d.ts +18 -0
- package/dist/schemas/registry.js +2 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/sdcpp-config.d.ts +75 -8
- package/dist/schemas/sdcpp-config.js +56 -10
- package/dist/schemas/sdk-config.d.ts +342 -0
- package/dist/schemas/text-to-speech.d.ts +8 -0
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription.d.ts +12 -0
- package/dist/schemas/transcription.js +8 -0
- package/dist/schemas/translation-config.d.ts +44 -16
- package/dist/schemas/translation-config.js +2 -0
- package/dist/schemas/vla.d.ts +14 -0
- package/dist/schemas/vla.js +26 -3
- package/dist/server/bare/delegate-connect-diagnostics.d.ts +9 -0
- package/dist/server/bare/delegate-connect-diagnostics.js +44 -0
- package/dist/server/bare/delegate-rpc-client.js +10 -12
- package/dist/server/bare/hyperswarm.js +1 -0
- package/dist/server/bare/ops/bci-transcribe.d.ts +13 -0
- package/dist/server/bare/ops/bci-transcribe.js +248 -0
- package/dist/server/bare/ops/embed.js +2 -1
- package/dist/server/bare/ops/kv-cache-utils.js +4 -3
- package/dist/server/bare/ops/transcribe.js +6 -2
- package/dist/server/bare/ops/translate.js +1 -1
- package/dist/server/bare/plugins/bci-whispercpp-transcription/plugin.d.ts +130 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/plugin.js +116 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-run.js +4 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.d.ts +7 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.js +1 -1
- package/dist/server/bare/plugins/index.d.ts +1 -0
- package/dist/server/bare/plugins/index.js +1 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/finetune.js +22 -29
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +2 -0
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +38 -7
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.d.ts +1 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +9 -1
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +14 -0
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +2 -0
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +8 -0
- package/dist/server/bare/plugins/sdcpp-generation/ops/video.js +7 -1
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +58 -4
- package/dist/server/bare/plugins/sdcpp-generation/plugin.js +9 -4
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +10 -0
- package/dist/server/bare/registry/model-config-utils.js +3 -0
- package/dist/server/bare/runtime/request-registry-singleton.js +11 -4
- package/dist/server/bare/runtime/request-registry.d.ts +60 -16
- package/dist/server/bare/runtime/request-registry.js +258 -15
- package/dist/server/bare/runtime/with-request-context.d.ts +1 -1
- package/dist/server/bare/runtime/with-request-context.js +1 -1
- package/dist/server/bare/types/addon-responses.d.ts +5 -0
- package/dist/server/rpc/handler-registry.js +4 -0
- package/dist/server/rpc/handlers/bci-transcribe-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/bci-transcribe-stream.js +4 -0
- package/dist/server/rpc/handlers/bci-transcribe.d.ts +3 -0
- package/dist/server/rpc/handlers/bci-transcribe.js +4 -0
- package/dist/server/rpc/handlers/download-asset.js +1 -1
- package/dist/server/rpc/handlers/get-model-info.js +3 -3
- package/dist/server/rpc/handlers/index.d.ts +4 -0
- package/dist/server/rpc/handlers/index.js +4 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/server/rpc/handlers/load-model/handler.js +1 -1
- package/dist/server/rpc/handlers/load-model/http.js +6 -6
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +1 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.d.ts +1 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.js +2 -2
- package/dist/server/rpc/handlers/load-model/registry.js +1 -0
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/server/rpc/handlers/rag.js +5 -5
- package/dist/server/utils/checksum.js +2 -1
- package/dist/server/utils/formatting.js +2 -1
- package/dist/server/utils/multi-gpu-mobile.d.ts +5 -0
- package/dist/server/utils/multi-gpu-mobile.js +10 -0
- package/dist/server/worker.js +2 -1
- package/dist/utils/aggregate-events.d.ts +2 -1
- package/dist/utils/aggregate-events.js +11 -3
- package/dist/utils/errors-server.d.ts +6 -4
- package/dist/utils/errors-server.js +6 -4
- package/electron-forge/index.cjs +820 -0
- package/package.json +73 -35
- package/dist/scripts/run-security-tests-bare.d.ts +0 -2
- package/dist/scripts/run-security-tests-bare.js +0 -78
|
@@ -5,7 +5,6 @@ import { loadModel, translate, unloadModel, MARIAN_EN_HI_INDIC_1B_Q4_0, } from "
|
|
|
5
5
|
try {
|
|
6
6
|
const modelId = await loadModel({
|
|
7
7
|
modelSrc: MARIAN_EN_HI_INDIC_1B_Q4_0,
|
|
8
|
-
modelType: "nmt",
|
|
9
8
|
onProgress: (progress) => {
|
|
10
9
|
console.log(progress);
|
|
11
10
|
},
|
|
@@ -20,7 +19,7 @@ try {
|
|
|
20
19
|
const result = translate({
|
|
21
20
|
modelId,
|
|
22
21
|
text,
|
|
23
|
-
modelType: "
|
|
22
|
+
modelType: "nmtcpp-translation",
|
|
24
23
|
stream: false,
|
|
25
24
|
});
|
|
26
25
|
const translatedText = await result.text;
|
|
@@ -2,7 +2,6 @@ import { translate, loadModel, unloadModel, AFRICAN_4B_TRANSLATION_Q4_K_M, } fro
|
|
|
2
2
|
try {
|
|
3
3
|
const modelId = await loadModel({
|
|
4
4
|
modelSrc: AFRICAN_4B_TRANSLATION_Q4_K_M,
|
|
5
|
-
modelType: "llm",
|
|
6
5
|
onProgress: (progress) => {
|
|
7
6
|
console.log(progress);
|
|
8
7
|
},
|
|
@@ -16,8 +15,8 @@ try {
|
|
|
16
15
|
repeat_penalty: 1,
|
|
17
16
|
seed: 42,
|
|
18
17
|
predict: 256,
|
|
19
|
-
stop_sequences: [
|
|
20
|
-
}
|
|
18
|
+
stop_sequences: ["\n"],
|
|
19
|
+
},
|
|
21
20
|
});
|
|
22
21
|
// With explicit source language
|
|
23
22
|
const engText = "Hello, how are you today?";
|
|
@@ -26,7 +25,7 @@ try {
|
|
|
26
25
|
text: engText,
|
|
27
26
|
from: "en",
|
|
28
27
|
to: "swh_Latn",
|
|
29
|
-
modelType: "
|
|
28
|
+
modelType: "llamacpp-completion",
|
|
30
29
|
stream: false,
|
|
31
30
|
});
|
|
32
31
|
const translatedTextExplicit = await resultExplicit.text;
|
|
@@ -2,7 +2,6 @@ import { translate, loadModel, unloadModel, SALAMANDRATA_2B_INST_Q8, } from "@qv
|
|
|
2
2
|
try {
|
|
3
3
|
const modelId = await loadModel({
|
|
4
4
|
modelSrc: SALAMANDRATA_2B_INST_Q8,
|
|
5
|
-
modelType: "llm",
|
|
6
5
|
onProgress: (progress) => {
|
|
7
6
|
console.log(progress);
|
|
8
7
|
},
|
|
@@ -15,7 +14,7 @@ try {
|
|
|
15
14
|
text: engText,
|
|
16
15
|
from: "en",
|
|
17
16
|
to: "es",
|
|
18
|
-
modelType: "
|
|
17
|
+
modelType: "llamacpp-completion",
|
|
19
18
|
context: "Use formal language, letter for financial institution",
|
|
20
19
|
stream: false,
|
|
21
20
|
});
|
|
@@ -2,7 +2,6 @@ import { translate, loadModel, unloadModel, SALAMANDRATA_2B_INST_Q4, } from "@qv
|
|
|
2
2
|
try {
|
|
3
3
|
const modelId = await loadModel({
|
|
4
4
|
modelSrc: SALAMANDRATA_2B_INST_Q4,
|
|
5
|
-
modelType: "llm",
|
|
6
5
|
onProgress: (progress) => {
|
|
7
6
|
console.log(progress);
|
|
8
7
|
},
|
|
@@ -14,7 +13,7 @@ try {
|
|
|
14
13
|
text: engText,
|
|
15
14
|
from: "en",
|
|
16
15
|
to: "it",
|
|
17
|
-
modelType: "
|
|
16
|
+
modelType: "llamacpp-completion",
|
|
18
17
|
stream: false,
|
|
19
18
|
});
|
|
20
19
|
const translatedTextExplicit = await resultExplicit.text;
|
|
@@ -24,7 +23,7 @@ try {
|
|
|
24
23
|
modelId,
|
|
25
24
|
text: espText,
|
|
26
25
|
to: "en",
|
|
27
|
-
modelType: "
|
|
26
|
+
modelType: "llamacpp-completion",
|
|
28
27
|
stream: false,
|
|
29
28
|
});
|
|
30
29
|
const translatedTextAutodetect = await resultAutodetect.text;
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { loadModel, translate, unloadModel, BERGAMOT_EN_ES
|
|
1
|
+
import { loadModel, translate, unloadModel, BERGAMOT_EN_ES } from "@qvac/sdk";
|
|
2
2
|
try {
|
|
3
3
|
const modelId = await loadModel({
|
|
4
4
|
modelSrc: BERGAMOT_EN_ES,
|
|
5
|
-
modelType: "nmt",
|
|
6
5
|
modelConfig: {
|
|
7
6
|
engine: "Bergamot",
|
|
8
7
|
from: "en",
|
|
@@ -15,7 +14,7 @@ try {
|
|
|
15
14
|
const streamResult = translate({
|
|
16
15
|
modelId,
|
|
17
16
|
text,
|
|
18
|
-
modelType: "
|
|
17
|
+
modelType: "nmtcpp-translation",
|
|
19
18
|
stream: true,
|
|
20
19
|
});
|
|
21
20
|
process.stdout.write("Translated text EN -> ES: ");
|
|
@@ -7,8 +7,7 @@ const [referenceAudioSrc] = process.argv.slice(2);
|
|
|
7
7
|
const CHATTERBOX_SAMPLE_RATE = 24000;
|
|
8
8
|
try {
|
|
9
9
|
const modelId = await loadModel({
|
|
10
|
-
modelSrc: TTS_T3_TURBO_EN_CHATTERBOX_Q8_0
|
|
11
|
-
modelType: "tts",
|
|
10
|
+
modelSrc: TTS_T3_TURBO_EN_CHATTERBOX_Q8_0,
|
|
12
11
|
modelConfig: {
|
|
13
12
|
ttsEngine: "chatterbox",
|
|
14
13
|
language: "en",
|
|
@@ -59,8 +59,7 @@ function appendPcmSamples(target, chunk) {
|
|
|
59
59
|
try {
|
|
60
60
|
console.log(`Loading LLM from registry: ${LLAMA_3_2_1B_INST_Q4_0.name}`);
|
|
61
61
|
const llmModelId = await loadModel({
|
|
62
|
-
modelSrc: LLAMA_3_2_1B_INST_Q4_0
|
|
63
|
-
modelType: "llm",
|
|
62
|
+
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
64
63
|
modelConfig: {
|
|
65
64
|
ctx_size: 2048,
|
|
66
65
|
},
|
|
@@ -69,8 +68,7 @@ try {
|
|
|
69
68
|
console.log(`LLM ready: ${llmModelId}`);
|
|
70
69
|
console.log("Loading Supertonic TTS (registry)…");
|
|
71
70
|
const ttsModelId = await loadModel({
|
|
72
|
-
modelSrc: TTS_EN_SUPERTONIC_Q8_0
|
|
73
|
-
modelType: "tts",
|
|
71
|
+
modelSrc: TTS_EN_SUPERTONIC_Q8_0,
|
|
74
72
|
modelConfig: {
|
|
75
73
|
ttsEngine: "supertonic",
|
|
76
74
|
language: "en",
|
|
@@ -4,8 +4,7 @@ import { createWav, playAudio, int16ArrayToBuffer, createWavHeader, } from "./ut
|
|
|
4
4
|
const SUPERTONIC_SAMPLE_RATE = 44100;
|
|
5
5
|
try {
|
|
6
6
|
const modelId = await loadModel({
|
|
7
|
-
modelSrc: TTS_MULTILINGUAL_SUPERTONIC2_Q8_0
|
|
8
|
-
modelType: "tts",
|
|
7
|
+
modelSrc: TTS_MULTILINGUAL_SUPERTONIC2_Q8_0,
|
|
9
8
|
modelConfig: {
|
|
10
9
|
ttsEngine: "supertonic",
|
|
11
10
|
language: "es",
|
|
@@ -5,8 +5,7 @@ import { createWav, playAudio, int16ArrayToBuffer, createWavHeader, } from "./ut
|
|
|
5
5
|
const SUPERTONIC_SAMPLE_RATE = 44100;
|
|
6
6
|
try {
|
|
7
7
|
const modelId = await loadModel({
|
|
8
|
-
modelSrc: TTS_EN_SUPERTONIC_Q8_0
|
|
9
|
-
modelType: "tts",
|
|
8
|
+
modelSrc: TTS_EN_SUPERTONIC_Q8_0,
|
|
10
9
|
modelConfig: {
|
|
11
10
|
ttsEngine: "supertonic",
|
|
12
11
|
language: "en",
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* π₀.₅ (pi05) vision-language-action example using the QVAC SDK.
|
|
3
|
+
*
|
|
4
|
+
* Loads the Physical Intelligence π₀.₅ GGUF model, runs a single inference
|
|
5
|
+
* pass with synthetic inputs (zero-filled gray images + BOS-only tokens +
|
|
6
|
+
* seeded noise), and prints the produced action chunk + per-stage timings.
|
|
7
|
+
*
|
|
8
|
+
* π₀.₅ differs from SmolVLA in two ways the SDK surfaces via `vlaHparams()`:
|
|
9
|
+
* - `numCameras: 3` — it expects exactly three camera frames (not two).
|
|
10
|
+
* - `stateInputMode: 'discrete'` — the robot state is tokenised into the
|
|
11
|
+
* language prompt, so the `state` buffer is ignored. We pass an empty
|
|
12
|
+
* `Float32Array(0)`. π₀.₅ also requires the `noise` prior.
|
|
13
|
+
*
|
|
14
|
+
* Usage:
|
|
15
|
+
* bun examples/vla-pi05.ts [path-to-pi05.gguf]
|
|
16
|
+
*
|
|
17
|
+
* By default the example pulls the registry-baked π₀.₅ GGUF (~3.9 GB) on
|
|
18
|
+
* first run and caches it locally. Pass an absolute path on the command line
|
|
19
|
+
* to override and load a local GGUF instead.
|
|
20
|
+
*/
|
|
21
|
+
import { close, loadModel, PI05_BASE_Q_AGGRESSIVE, unloadModel, vla, vlaHparams, vlaPreprocessImage, } from "@qvac/sdk";
|
|
22
|
+
const modelSrcOverride = process.argv[2];
|
|
23
|
+
const modelSrc = modelSrcOverride ?? PI05_BASE_Q_AGGRESSIVE;
|
|
24
|
+
try {
|
|
25
|
+
console.log("Loading π₀.₅ (pi05) model...");
|
|
26
|
+
const modelId = await loadModel({
|
|
27
|
+
modelSrc,
|
|
28
|
+
modelType: "ggml-vla",
|
|
29
|
+
modelConfig: { backend: "cpu" },
|
|
30
|
+
onProgress: (p) => typeof modelSrc === "string"
|
|
31
|
+
? undefined
|
|
32
|
+
: process.stdout.write(`\rDownloading: ${p.percentage.toFixed(1)}%`),
|
|
33
|
+
});
|
|
34
|
+
if (typeof modelSrc !== "string")
|
|
35
|
+
process.stdout.write("\n");
|
|
36
|
+
console.log(`Model loaded: ${modelId}`);
|
|
37
|
+
const { hparams, backendName } = await vlaHparams({ modelId });
|
|
38
|
+
console.log(`Backend: ${backendName ?? "(unknown)"}`);
|
|
39
|
+
console.log("Hparams:", hparams);
|
|
40
|
+
// Build synthetic inputs sized to the model's expectations. A real
|
|
41
|
+
// consumer would: read camera frames, tokenize the instruction with the
|
|
42
|
+
// model's tokenizer, and (for π₀.₅) inline the robot state into the prompt.
|
|
43
|
+
const size = hparams.visionImageSize;
|
|
44
|
+
const numCameras = hparams.numCameras ?? 3;
|
|
45
|
+
const dummyPixels = new Uint8Array(size * size * 3).fill(128);
|
|
46
|
+
// π₀.₅ expects exactly `numCameras` frames.
|
|
47
|
+
const images = Array.from({ length: numCameras }, () => vlaPreprocessImage(dummyPixels, size, size, { size }));
|
|
48
|
+
const tokens = new Int32Array(hparams.tokenizerMaxLength);
|
|
49
|
+
const mask = new Uint8Array(hparams.tokenizerMaxLength);
|
|
50
|
+
// BOS-only "instruction" for the smoke test.
|
|
51
|
+
tokens[0] = 1;
|
|
52
|
+
mask[0] = 1;
|
|
53
|
+
// Discrete-state model: the state buffer is ignored (state is tokenised
|
|
54
|
+
// into the prompt), so pass an empty Float32Array. π₀.₅ requires `noise`.
|
|
55
|
+
const state = new Float32Array(0);
|
|
56
|
+
const noise = new Float32Array(hparams.chunkSize * hparams.maxActionDim);
|
|
57
|
+
console.log("Running VLA inference...");
|
|
58
|
+
const { actions, actionDim, chunkSize, stats } = await vla({
|
|
59
|
+
modelId,
|
|
60
|
+
images,
|
|
61
|
+
imgWidth: size,
|
|
62
|
+
imgHeight: size,
|
|
63
|
+
state,
|
|
64
|
+
tokens,
|
|
65
|
+
mask,
|
|
66
|
+
noise,
|
|
67
|
+
});
|
|
68
|
+
console.log(`Got ${chunkSize} action steps of dim ${actionDim}.`);
|
|
69
|
+
console.log("First step:", Array.from(actions.subarray(0, actionDim)));
|
|
70
|
+
if (stats) {
|
|
71
|
+
console.log(`Timing: vision=${stats.vision_ms?.toFixed(0)}ms ` +
|
|
72
|
+
`prefill=${stats.prefill_total_ms?.toFixed(0)}ms ` +
|
|
73
|
+
`ode=${stats.ode_ms?.toFixed(0)}ms ` +
|
|
74
|
+
`total=${stats.total_ms?.toFixed(0)}ms`);
|
|
75
|
+
}
|
|
76
|
+
await unloadModel({ modelId, clearStorage: false });
|
|
77
|
+
console.log("Model unloaded.");
|
|
78
|
+
process.exit(0);
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
console.error("π₀.₅ example failed:", error);
|
|
82
|
+
await close();
|
|
83
|
+
process.exit(1);
|
|
84
|
+
}
|
|
@@ -76,7 +76,6 @@ for (const tool of ["ffmpeg", "ffplay"]) {
|
|
|
76
76
|
console.log("Loading whisper-tiny + Silero VAD...");
|
|
77
77
|
const asrModelId = await loadModel({
|
|
78
78
|
modelSrc: WHISPER_TINY,
|
|
79
|
-
modelType: "whisper",
|
|
80
79
|
modelConfig: {
|
|
81
80
|
vadModelSrc: VAD_SILERO_5_1_2,
|
|
82
81
|
audio_format: "f32le",
|
|
@@ -93,15 +92,13 @@ const asrModelId = await loadModel({
|
|
|
93
92
|
console.log("Loading Llama 3.2 1B...");
|
|
94
93
|
const llmModelId = await loadModel({
|
|
95
94
|
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
96
|
-
modelType: "llm",
|
|
97
95
|
modelConfig: {
|
|
98
96
|
ctx_size: 4096,
|
|
99
97
|
},
|
|
100
98
|
});
|
|
101
99
|
console.log("Loading Supertonic TTS...");
|
|
102
100
|
const ttsModelId = await loadModel({
|
|
103
|
-
modelSrc: TTS_EN_SUPERTONIC_Q8_0
|
|
104
|
-
modelType: "tts",
|
|
101
|
+
modelSrc: TTS_EN_SUPERTONIC_Q8_0,
|
|
105
102
|
modelConfig: {
|
|
106
103
|
ttsEngine: "supertonic",
|
|
107
104
|
language: "en",
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import "#polyfill-bare-globals";
|
|
2
|
-
export { completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, loggingStream, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, classify, video, type VideoProgressTick, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle, } from "./client/api/index";
|
|
2
|
+
export { completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, loggingStream, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, classify, video, type VideoProgressTick, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle, } from "./client/api/index";
|
|
3
3
|
export { close } from "./client/index";
|
|
4
4
|
export { plugins } from "./client/plugins-factory";
|
|
5
|
-
export { type LifecycleState, type ModelProgressUpdate, type LoadModelOptions, type LoadCustomPluginModelOptions, type DownloadAssetOptions, type Tool, type ToolCall, type ToolCallWithCall, type ToolCallError, type ToolCallEvent, type CompletionEvent, type CompletionFinal, type CompletionRun, type CompletionStats, type EmbedStats, VERBOSITY, type Attachment, type TranscribeStreamSession, type TranscribeStreamMetadataSession, type TranscribeStreamConversationSession, type TranscribeStreamEvent, type VadStateEvent, type EndOfTurnEvent, type TranscribeSegment, type TextToSpeechStreamSession, type TextToSpeechStreamResponse, type TextToSpeechStreamClientParams, type CompletionParams, type ToolDialect, type RagSearchResult, type RagSaveEmbeddingsResult, type RagReindexResult, type RagEmbeddedDoc, type RagDoc, type RagWorkspaceInfo, type RagCloseWorkspaceParams, type RagDeleteWorkspaceParams, type RagIngestStage, type RagReindexStage, type RagSaveStage, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, type QvacConfig, type ModelInfo, type GetModelInfoParams, type GetLoadedModelInfoParams, type LoadedModelInfo, type LoadedInstance, type CacheFileInfo, toolSchema, TOOLS_MODE, type ToolsMode, type McpClient, type McpClientInput, type OCRClientParams, type OCRTextBlock, type OCROptions, type ClassifyClientParams, type ClassificationResult, type DiffusionClientParams, type DiffusionStreamResponse, type DiffusionStats, type VideoClientParams, type VideoStreamResponse, type VideoStats, type UpscaleClientParams, type UpscaleStreamResponse, type UpscaleStats, type VlaConfig, type VlaClientRunParams, type VlaClientRunResult, type VlaHparams, type VlaStats, definePlugin, defineHandler, defineDuplexHandler, type QvacPlugin, type CreateModelParams, type PluginModelResult, type ModelRegistryEntry, type ModelRegistryEntryAddon, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS, type BuiltinPlugin, type ProfilerMode, type FinetuneValidation, type FinetuneRunParams, type FinetuneGetStateParams, type FinetuneStopParams, type FinetuneParams, type FinetuneStatus, type FinetuneProgress, type FinetuneStats, type FinetuneResult, } from "./schemas/index";
|
|
5
|
+
export { type LifecycleState, type ModelProgressUpdate, type LoadModelOptions, type LoadCustomPluginModelOptions, type DownloadAssetOptions, type Tool, type ToolCall, type ToolCallWithCall, type ToolCallError, type ToolCallEvent, type CompletionEvent, type CompletionFinal, type CompletionRun, type CompletionStats, type EmbedStats, VERBOSITY, type Attachment, type TranscribeStreamSession, type TranscribeStreamMetadataSession, type TranscribeStreamConversationSession, type TranscribeStreamEvent, type VadStateEvent, type EndOfTurnEvent, type TranscribeSegment, type BciConfig, type BciTranscribeClientParams, type BciTranscribeStreamClientParams, type BciTranscribeStreamSession, type BciTranscribeStreamMetadataSession, type BciStreamOpts, type NeuralInput, type TextToSpeechStreamSession, type TextToSpeechStreamResponse, type TextToSpeechStreamClientParams, type CompletionParams, type ToolDialect, type RagSearchResult, type RagSaveEmbeddingsResult, type RagReindexResult, type RagEmbeddedDoc, type RagDoc, type RagWorkspaceInfo, type RagCloseWorkspaceParams, type RagDeleteWorkspaceParams, type RagIngestStage, type RagReindexStage, type RagSaveStage, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, type QvacConfig, type ModelInfo, type GetModelInfoParams, type GetLoadedModelInfoParams, type LoadedModelInfo, type LoadedInstance, type CacheFileInfo, toolSchema, TOOLS_MODE, type ToolsMode, type McpClient, type McpClientInput, type OCRClientParams, type OCRTextBlock, type OCROptions, type ClassifyClientParams, type ClassificationResult, type DiffusionClientParams, type DiffusionStreamResponse, type DiffusionStats, type VideoClientParams, type VideoStreamResponse, type VideoStats, type UpscaleClientParams, type UpscaleStreamResponse, type UpscaleStats, type VlaConfig, type VlaClientRunParams, type VlaClientRunResult, type VlaHparams, type VlaStats, definePlugin, defineHandler, defineDuplexHandler, type QvacPlugin, type CreateModelParams, type PluginModelResult, type ModelRegistryEntry, type ModelRegistryEntryAddon, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS, type BuiltinPlugin, type ProfilerMode, type FinetuneValidation, type FinetuneRunParams, type FinetuneGetStateParams, type FinetuneStopParams, type FinetuneParams, type FinetuneStatus, type FinetuneProgress, type FinetuneStats, type FinetuneResult, } from "./schemas/index";
|
|
6
6
|
export { type ToolInput, type ToolHandler } from "./utils/tool-helpers";
|
|
7
7
|
export { MODEL_TYPES, ModelType } from "./schemas/index";
|
|
8
8
|
export * from "./models/registry/index";
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import "#polyfill-bare-globals";
|
|
2
2
|
// Public API exports only
|
|
3
|
-
export { completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, loggingStream, ocr, invokePlugin, invokePluginStream, diffusion, classify, video, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, } from "./client/api/index.js";
|
|
3
|
+
export { completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, loggingStream, ocr, invokePlugin, invokePluginStream, diffusion, classify, video, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, } from "./client/api/index.js";
|
|
4
4
|
export { close } from "./client/index.js";
|
|
5
5
|
export { plugins } from "./client/plugins-factory.js";
|
|
6
|
-
export { VERBOSITY, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, toolSchema, TOOLS_MODE, definePlugin, defineHandler, defineDuplexHandler, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS, } from "./schemas/index.js";
|
|
6
|
+
export { VERBOSITY, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, toolSchema, TOOLS_MODE, definePlugin, defineHandler, defineDuplexHandler, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS, } from "./schemas/index.js";
|
|
7
7
|
export {} from "./utils/tool-helpers.js";
|
|
8
8
|
// Model types - canonical naming with backward-compatible aliases
|
|
9
9
|
export { MODEL_TYPES, ModelType } from "./schemas/index.js";
|
|
@@ -15,13 +15,15 @@ export { SUPPORTED_AUDIO_FORMATS } from "./constants/audio.js";
|
|
|
15
15
|
// envelope, but consumers reach for it through `instanceof` on
|
|
16
16
|
// `await run.final` / `run.text` / `run.toolCalls` / `run.stats`
|
|
17
17
|
// rejections. `RequestRejectedByPolicyError` is thrown by
|
|
18
|
-
// `RequestRegistry.begin(...)` when a registered concurrency policy
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
18
|
+
// `await RequestRegistry.begin(...)` when a registered concurrency policy
|
|
19
|
+
// refuses a new request. With the default queue policy a same-model
|
|
20
|
+
// `completion` no longer rejects — it waits FIFO — so this now surfaces the
|
|
21
|
+
// bounded-queue cases: `onOverflow: "reject"`, the per-model queue-depth cap,
|
|
22
|
+
// or a `queueTimeoutMs` elapsing. It propagates out through the worker so the
|
|
23
|
+
// client can distinguish "the model is saturated" from "the request failed".
|
|
22
24
|
//
|
|
23
25
|
// `RequestIdConflictError` and `RequestNotFoundError` are thrown by
|
|
24
|
-
// `RequestRegistry.begin(...)` / `.end(...)` on UUID collisions and
|
|
26
|
+
// `await RequestRegistry.begin(...)` / `.end(...)` on UUID collisions and
|
|
25
27
|
// missing-target cancels. They're surfaced here so consumers using
|
|
26
28
|
// the decorated-promise `requestId` can pattern-match on rejected
|
|
27
29
|
// cancel paths. All three classes round-trip the RPC boundary via
|