@qvac/sdk 0.17.0 → 0.18.0

package/dist/examples/multi-job-completion.js

@@ -0,0 +1,128 @@
+/**
+ * Multi-job continuous batching — concurrent completions on one loaded LLM.
+ *
+ * Unlike `batchCompletion()` (which bundles many prompts into ONE call), this
+ * shows independent `completion()` calls fired concurrently against the SAME
+ * loaded model. When the model is loaded with `modelConfig.parallel >= 2`, the
+ * SDK admits up to `parallel` completions at once and the llama.cpp addon
+ * decodes their sequences together — so a burst of prompts finishes in roughly
+ * the time of the slowest one, not the sum of all of them.
+ *
+ * Each `completion()` returns an independent `CompletionRun`:
+ *  - `events`     — an `AsyncIterable` of streamed events for that call.
+ *  - `final`      — a `Promise<CompletionFinal>` with THIS call's own
+ *                   `contentText` and `stats` (its own tokens/sec, TTFT).
+ *                   `stats.avgConcurrentSeq` stays model-level (how busy the
+ *                   shared backend was).
+ *  - `requestId`  — cancel just this call with `cancel({ requestId })`; peers
+ *                   keep decoding.
+ *
+ * Non-cached completions and batches share the model's `parallel` admission
+ * cap: with `parallel` in flight, admitting one more waits FIFO until a slot
+ * frees.
+ *
+ * Run from packages/sdk:
+ *   bun run examples/multi-job-completion.ts
+ */
+import { completion, cancel, loadModel, unloadModel, InferenceCancelledError, LLAMA_3_2_1B_INST_Q4_0 } from '@qvac/sdk';
+const PROMPTS = [
+    { id: 'cherry', ask: 'Reply with only the word CHERRY.' },
+    { id: 'banana', ask: 'Reply with only the word BANANA.' },
+    { id: 'grape', ask: 'Reply with only the word GRAPE.' },
+    { id: 'lemon', ask: 'Reply with only the word LEMON.' }
+];
+try {
+    // `parallel: 4` opens the concurrent decode slots continuous batching needs.
+    const modelId = await loadModel({
+        modelSrc: LLAMA_3_2_1B_INST_Q4_0,
+        modelType: 'llm',
+        modelConfig: { ctx_size: 4096, parallel: 4 },
+        onProgress: (p) => {
+            const mb = (n) => (n / 1e6).toFixed(1);
+            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
+            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
+            if (p.percentage >= 100)
+                process.stderr.write('\n');
+        }
+    });
+    console.log(`▸ Model loaded: ${modelId}`);
+    // ---- Concurrent completions. Fire every call before awaiting any, so all
+    // four are admitted and decode together (not one-after-another). ----
+    console.log('\n▸ Firing 4 completions concurrently:');
+    const started = Date.now();
+    const runs = PROMPTS.map(({ ask }) => completion({
+        modelId,
+        history: [{ role: 'user', content: ask }],
+        generationParams: { temp: 0, seed: 42, predict: 16 },
+        stream: true
+    }));
+    const outputs = await Promise.all(runs.map(async (run, i) => {
+        const final = await run.final;
+        return {
+            id: PROMPTS[i].id,
+            text: final.contentText.replace(/\s+/g, ' ').trim(),
+            stats: final.stats
+        };
+    }));
+    const elapsed = Date.now() - started;
+    for (const { id, text, stats } of outputs) {
+        const tps = stats?.tokensPerSecond?.toFixed(1) ?? 'n/a';
+        console.log(`  ▸ ${id}: "${text}"  (own ${tps} tok/s)`);
+    }
+    // Report the engine's own measure of overlap rather than inferring it from
+    // wall-clock: avgConcurrentSeq > 1 means the addon decoded sequences together.
+    const peakConcurrent = Math.max(...outputs.map((o) => o.stats?.avgConcurrentSeq ?? 0));
+    console.log(`\n▸ All 4 finished in ${elapsed} ms; engine avg concurrent sequences: ${peakConcurrent.toFixed(1)} (> 1 = decoded together).`);
+    // ---- Per-request cancel isolation. Fire two long runs; cancel only the
+    // first by its requestId. The second must keep decoding past the cancel. ----
+    console.log('\n▸ Per-request cancel: cancelling one of two in-flight completions:');
+    const doomed = completion({
+        modelId,
+        history: [{ role: 'user', content: 'Write a long story about an otter.' }],
+        generationParams: { temp: 0, seed: 1, predict: 256 },
+        stream: true
+    });
+    const survivor = completion({
+        modelId,
+        history: [{ role: 'user', content: 'Write a long story about a melon.' }],
+        generationParams: { temp: 0, seed: 42, predict: 256 },
+        stream: true
+    });
+    // Consume the survivor's stream directly so we can count tokens produced AFTER
+    // the cancel is acknowledged — a short survivor could otherwise finish before
+    // the cancel lands and pass even under a model-wide cancel.
+    const survivorTokens = survivor.events[Symbol.asyncIterator]();
+    // Wait until BOTH runs are actually streaming before cancelling, so the cancel
+    // hits a genuinely in-flight peer, not a pre-admission drop.
+    await Promise.all([doomed.events[Symbol.asyncIterator]().next(), survivorTokens.next()]);
+    await cancel({ requestId: doomed.requestId });
+    let doomedCancelled = false;
+    try {
+        await doomed.final;
+    }
+    catch (err) {
+        doomedCancelled = err instanceof InferenceCancelledError;
+    }
+    // Drain what the survivor emits from here on: a per-request cancel leaves it
+    // decoding to completion, a model-wide cancel would have stopped it.
+    let tokensAfterCancel = 0;
+    for (let next = await survivorTokens.next(); !next.done; next = await survivorTokens.next()) {
+        tokensAfterCancel++;
+    }
+    const survivorText = (await survivor.final).contentText.replace(/\s+/g, ' ').trim();
+    console.log(`  ▸ cancelled run -> ${doomedCancelled ? 'cancelled' : 'NOT cancelled'}`);
+    console.log(`  ▸ peer kept decoding -> +${tokensAfterCancel} tokens after the cancel ack`);
+    await unloadModel({ modelId, clearStorage: false });
+    // A zero exit must prove concurrent scheduling, typed cancellation, and survivor
+    // isolation: the first burst reported multiple resident sequences, the doomed
+    // run cancelled, and its peer produced tokens AFTER the cancel landed.
+    if (peakConcurrent <= 1 || !doomedCancelled || tokensAfterCancel === 0) {
+        console.error(`✖ multi-job behavior not demonstrated (peakConcurrent=${peakConcurrent.toFixed(2)}, doomedCancelled=${doomedCancelled}, tokensAfterCancel=${tokensAfterCancel}, survivorText="${survivorText}")`);
+        process.exit(1);
+    }
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    process.exit(1);
+}

package/dist/examples/profiling/basic.js

@@ -3,7 +3,8 @@ try {
     // Enable profiling globally
     profiler.enable({
         mode: 'verbose',
-        includeServerBreakdown: true
+        includeServerBreakdown: true,
+        includeResourceGauges: true
     });
     console.log('▸ Profiler enabled:', profiler.isEnabled());
     const modelId = await loadModel({
@@ -62,6 +63,36 @@ try {
         const ops = [...new Set(json.recentEvents?.map((e) => `${e.op}:${e.kind}`) ?? [])];
         console.log('  Available ops:', ops.join(', '));
     }
+    const resourceEvent = json.recentEvents?.filter((event) => event.resources).at(-1);
+    console.log('\n▸ Resource Gauges');
+    if (resourceEvent?.resources) {
+        const resources = resourceEvent.resources;
+        console.log('  op:', resourceEvent.op);
+        console.log('  origin:', resources.origin);
+        console.log('  cpu:', resources.cpu.status === 'supported'
+            ? `${(resources.cpu.value * 100).toFixed(1)}%`
+            : resources.cpu.status);
+        console.log('  memoryUsed:', resources.memory.usedBytes.status === 'supported'
+            ? `${(resources.memory.usedBytes.value / 1024 ** 3).toFixed(2)} GiB`
+            : resources.memory.usedBytes.status);
+        console.log('  gpuCount:', resources.gpus.status === 'supported' ? resources.gpus.value.length : resources.gpus.status);
+    }
+    else {
+        console.log('  (no resource gauges reported)');
+    }
+    const backendEvent = json.recentEvents?.filter((event) => event.backend).at(-1);
+    console.log('\n▸ Backend Diagnostics');
+    if (backendEvent?.backend) {
+        const backend = backendEvent.backend;
+        console.log('  op:', backendEvent.op);
+        console.log('  selectedBackend:', backend.selectedBackend);
+        console.log('  selectedDevice:', backend.selectedDevice);
+        console.log('  graphicsApi:', backend.graphicsApi ?? '(not reported)');
+        console.log('  fallback:', backend.fallback?.reason ?? '(none)');
+    }
+    else {
+        console.log('  (no backend diagnostics reported for this operation)');
+    }
     console.log('\n▸ Profiler JSON (structure)');
     console.log('  aggregates:', Object.keys(json.aggregates).length, 'metrics');
     console.log('  recentEvents:', json.recentEvents?.length ?? 0, 'events');

package/dist/examples/system-resources.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=system-resources.d.ts.map

package/dist/examples/system-resources.js

@@ -0,0 +1,86 @@
+import { getSystemResources } from '@qvac/sdk';
+function formatBytes(bytes) {
+    return `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
+}
+function formatUtilization(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+function printMetric(label, metric, format, indent = '  ') {
+    if (metric.status === 'supported') {
+        const scope = metric.provenance.scope ? `/${metric.provenance.scope}` : '';
+        console.log(`${indent}${label}: ${format(metric.value)} (${metric.provenance.source}${scope})`);
+        return;
+    }
+    const reason = metric.reason ? ` — ${metric.reason}` : '';
+    console.log(`${indent}${label}: ${metric.status}${reason}`);
+}
+try {
+    // sample: true performs fresh usage queries, so request it only when current usage is needed.
+    const resources = await getSystemResources({ sample: true });
+    const { capabilities, sample } = resources;
+    console.log('▸ CPU');
+    if (capabilities.cpu.status === 'supported') {
+        const cpu = capabilities.cpu.value;
+        printMetric('Name', cpu.name, String);
+        printMetric('Physical cores', cpu.physicalCores, String);
+        printMetric('Logical cores', cpu.logicalCores, String);
+    }
+    else {
+        printMetric('Capabilities', capabilities.cpu, String);
+    }
+    console.log('\n▸ Memory');
+    printMetric('Total', capabilities.memory.totalBytes, formatBytes);
+    if (sample) {
+        printMetric('Used', sample.memory.usedBytes, formatBytes);
+        printMetric('Total (sampled)', sample.memory.totalBytes, formatBytes);
+    }
+    console.log('\n▸ GPUs');
+    const gpuLabels = new Map();
+    if (capabilities.gpus.status === 'supported') {
+        if (capabilities.gpus.value.length === 0) {
+            console.log('  No GPUs enumerated.');
+        }
+        for (const [index, gpu] of capabilities.gpus.value.entries()) {
+            const label = gpu.name.status === 'supported'
+                ? `${gpu.name.value} (GPU ${index + 1})`
+                : `GPU ${index + 1}`;
+            gpuLabels.set(gpu.id, label);
+            console.log(`  ${label}`);
+            console.log(`    ID: ${gpu.id}`);
+            printMetric('Vendor', gpu.vendor, String, '    ');
+            printMetric('Total memory', gpu.memoryTotalBytes, formatBytes, '    ');
+            const drivers = Object.entries(gpu.drivers)
+                .filter(([, metric]) => metric.status === 'supported' && metric.value)
+                .map(([driver]) => driver);
+            console.log(`    Graphics APIs: ${drivers.join(', ') || 'none reported'}`);
+        }
+    }
+    else {
+        printMetric('Capabilities', capabilities.gpus, String);
+    }
+    if (sample) {
+        console.log('\n▸ Live utilization');
+        printMetric('CPU', sample.cpu, formatUtilization);
+        if (sample.gpus.status === 'supported') {
+            if (sample.gpus.value.length === 0) {
+                console.log('  No GPU samples reported.');
+            }
+            for (const [index, gpu] of sample.gpus.value.entries()) {
+                const label = gpuLabels.get(gpu.id) ?? `GPU ${index + 1}`;
+                console.log(`  ${label}`);
+                console.log(`    ID: ${gpu.id}`);
+                printMetric('Compute', gpu.compute, formatUtilization, '    ');
+                printMetric('Memory used', gpu.memoryUsedBytes, formatBytes, '    ');
+            }
+        }
+        else {
+            printMetric('GPU samples', sample.gpus, String);
+        }
+    }
+    console.log('\n▸ Resource diagnostics do not reserve memory or guarantee model fit.');
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    process.exit(1);
+}

package/dist/examples/tts/audio8.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=audio8.d.ts.map

package/dist/examples/tts/audio8.js

@@ -0,0 +1,54 @@
+import { loadModel, textToSpeech, unloadModel, TTS_LM_MULTILINGUAL_AUDIO8_Q8_0, TTS_CODEC_DECODER_AUDIO8_Q8_0 } from '@qvac/sdk';
+import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './utils.js';
+// Audio8 (GGML): DualAR LM + codec decoder, native 44.1 kHz output. The
+// primary modelSrc is the LM GGUF; the codec decoder loads via modelConfig.
+// For zero-shot voice cloning also pass audio8CodecEncoderModelSrc plus
+// referenceAudioSrc/referenceText (a recording and its exact transcript).
+const AUDIO8_SAMPLE_RATE = 44100;
+try {
+    const modelId = await loadModel({
+        modelSrc: TTS_LM_MULTILINGUAL_AUDIO8_Q8_0,
+        modelConfig: {
+            ttsEngine: 'audio8',
+            audio8CodecDecoderModelSrc: TTS_CODEC_DECODER_AUDIO8_Q8_0,
+            temperature: 0.7,
+            topP: 0.9,
+            seed: 42
+        },
+        onProgress: (p) => {
+            const mb = (n) => (n / 1e6).toFixed(1);
+            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
+            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
+            if (p.percentage >= 100)
+                process.stderr.write('\n');
+        }
+    });
+    console.log(`▸ Model loaded: ${modelId}`);
+    console.log('▸ Testing Audio8 Text-to-Speech...');
+    const result = textToSpeech({
+        modelId,
+        text: 'Hey, how are you doing today?',
+        inputType: 'text',
+        stream: false
+    });
+    const audioBuffer = await result.buffer;
+    console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
+    console.log('▸ Saving audio to file...');
+    createWav(audioBuffer, AUDIO8_SAMPLE_RATE, 'audio8-output.wav');
+    console.log('▸ Audio saved to audio8-output.wav');
+    console.log('▸ Playing audio...');
+    const audioData = int16ArrayToBuffer(audioBuffer);
+    const wavBuffer = Buffer.concat([
+        createWavHeader(audioData.length, AUDIO8_SAMPLE_RATE),
+        audioData
+    ]);
+    playAudio(wavBuffer);
+    console.log('▸ Audio playback complete');
+    await unloadModel({ modelId });
+    console.log('▸ Model unloaded');
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    process.exit(1);
+}

package/dist/examples/tts/cosyvoice3.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cosyvoice3.d.ts.map

package/dist/examples/tts/cosyvoice3.js

@@ -0,0 +1,52 @@
+import { loadModel, textToSpeech, unloadModel, TTS_COSYVOICE3_LLM_COSYVOICE_Q8_0 } from '@qvac/sdk';
+import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './utils.js';
+// CosyVoice3 (GGML): zero-shot speech with one conditioning control per
+// synthesis (emotion, pace, or instruct). The LLM GGUF's registry companion
+// set downloads the flow/HiFT models, baked voice and tokenizer alongside it.
+// Native output is 24 kHz.
+const COSYVOICE3_SAMPLE_RATE = 24000;
+try {
+    const modelId = await loadModel({
+        modelSrc: TTS_COSYVOICE3_LLM_COSYVOICE_Q8_0,
+        modelConfig: {
+            ttsEngine: 'cosyvoice3',
+            seed: 42
+        },
+        onProgress: (p) => {
+            const mb = (n) => (n / 1e6).toFixed(1);
+            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
+            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
+            if (p.percentage >= 100)
+                process.stderr.write('\n');
+        }
+    });
+    console.log(`▸ Model loaded: ${modelId}`);
+    console.log('▸ Testing CosyVoice3 Text-to-Speech...');
+    const result = textToSpeech({
+        modelId,
+        text: 'Hey, how are you doing today?',
+        inputType: 'text',
+        stream: false,
+        emotion: 'happy'
+    });
+    const audioBuffer = await result.buffer;
+    console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
+    console.log('▸ Saving audio to file...');
+    createWav(audioBuffer, COSYVOICE3_SAMPLE_RATE, 'cosyvoice3-output.wav');
+    console.log('▸ Audio saved to cosyvoice3-output.wav');
+    console.log('▸ Playing audio...');
+    const audioData = int16ArrayToBuffer(audioBuffer);
+    const wavBuffer = Buffer.concat([
+        createWavHeader(audioData.length, COSYVOICE3_SAMPLE_RATE),
+        audioData
+    ]);
+    playAudio(wavBuffer);
+    console.log('▸ Audio playback complete');
+    await unloadModel({ modelId });
+    console.log('▸ Model unloaded');
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    process.exit(1);
+}

package/dist/examples/vla-groot.js

@@ -27,25 +27,48 @@
  * By default the example pulls the registry-baked GR00T-LIBERO GGUF (~3.76 GB)
  * on first run and caches it locally. Pass an absolute path on the command line
  * to override and load a local GGUF instead.
+ *
+ * Multi-embodiment GGUFs (vla-ggml >= 0.17.0): one GGUF can carry many
+ * embodiments, one active at a time. Optional env vars demonstrate selection:
+ *   QVAC_GROOT_EMBODIMENT        initial selection at load — a tag
+ *                                (e.g. 'libero_sim') or a numeric cat_id
+ *   QVAC_GROOT_SWITCH_EMBODIMENT after load, switch via vlaSetEmbodiment()
+ *                                and run inference on the new embodiment
+ * Both are no-ops you can omit; the GGUF's default embodiment is used. On a
+ * single-embodiment GGUF a switch is rejected by the addon.
  */
-import { close, loadModel, GROOT_Q8_VF16, unloadModel, vla, vlaHparams, vlaPadState } from '@qvac/sdk';
+import { close, loadModel, GROOT_Q8_VF16, unloadModel, vla, vlaHparams, vlaSetEmbodiment, vlaPadState } from '@qvac/sdk';
 // LIBERO GR00T prompt layout (Qwen3-VL tokenizer). GR00T reports
 // `hparams.tokenizerMaxLength === 0` — it does not surface a tokenizer length,
 // so the prompt length is fixed by the model: 2 cameras × 64 merged image
-// tokens (256 patches, 2×2 merge) = 128 image tokens, plus text, = 148 total.
+// tokens (256 patches, 2×2 merge) = 128 image tokens, plus text ≈ 150 total.
 // A real consumer gets these from the tokenizer; the smoke hard-codes them.
+// The prompt length follows the ACTIVE embodiment's camera count (a 4-camera
+// DROID row needs 4 image-token runs, which a fixed 2-camera length cannot
+// hold), so it is computed per inference, not a constant.
 const IMAGE_TOKEN_ID = 151655;
 const MERGED_TOKENS_PER_IMAGE = 64;
-const PROMPT_LENGTH = 148;
+const PROMPT_TEXT_TAIL = 20;
 const TEXT_TOKEN_ID = 1000;
 const modelSrcOverride = process.argv[2];
 const modelSrc = modelSrcOverride ?? GROOT_Q8_VF16;
+// A tag string or a numeric cat_id, straight from the env. '24' is a cat_id,
+// 'libero_sim' a tag.
+function embodimentFromEnv(name) {
+    const env = process.env;
+    const raw = env[name];
+    if (raw === undefined || raw === '')
+        return undefined;
+    return /^\d+$/.test(raw) ? Number(raw) : raw;
+}
+const embodiment = embodimentFromEnv('QVAC_GROOT_EMBODIMENT');
+const switchEmbodiment = embodimentFromEnv('QVAC_GROOT_SWITCH_EMBODIMENT');
 try {
     console.log('▸ Loading GR00T (N1.7-3B LIBERO) model...');
     const modelId = await loadModel({
         modelSrc,
         modelType: 'ggml-vla',
-        modelConfig: { backend: 'cpu' },
+        modelConfig: { backend: 'cpu', ...(embodiment !== undefined && { embodiment }) },
         onProgress: (p) => {
             const mb = (n) => (n / 1e6).toFixed(1);
             const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
@@ -60,54 +83,74 @@ try {
     const { hparams, backendName } = await vlaHparams({ modelId });
     console.log(`▸ Backend: ${backendName ?? '(unknown)'}`);
     console.log('▸ Hparams:', hparams);
-    const patchElems = hparams.imagePatchElems;
-    if (hparams.imageInputMode !== 'patches' || patchElems == null) {
-        throw new Error(`expected a patch-input model (imageInputMode 'patches' + imagePatchElems); ` +
-            `got imageInputMode=${hparams.imageInputMode}`);
+    if (hparams.selectedEmbodimentTag !== undefined) {
+        console.log(`▸ Embodiment: ${hparams.selectedEmbodimentTag} ` +
+            `(cat_id ${hparams.selectedEmbodimentCatId}, ${hparams.numCameras} cameras)`);
     }
-    const numCameras = hparams.numCameras ?? 2;
-    // Patch-input model: each camera is a pre-patchified buffer of exactly
-    // `imagePatchElems` floats. A real consumer patchifies each camera frame the
-    // way `Gr00tPolicy` does (the model's learned patch-embed runs inside the
-    // addon); we use small synthetic values.
-    const images = Array.from({ length: numCameras }, () => new Float32Array(patchElems).fill(0.02));
-    // Continuous-state model: pad the robot state to `maxStateDim`.
-    const state = vlaPadState([0, 0, 0, 0, 0, 0], hparams.maxStateDim);
-    // GR00T requires the noise prior (flow-matching; it is not sampled in-model).
-    const noise = new Float32Array(hparams.chunkSize * hparams.maxActionDim);
-    // Prompt: one run of `MERGED_TOKENS_PER_IMAGE` image tokens per camera, each
-    // followed by a text separator, padded out to the fixed `PROMPT_LENGTH`.
-    const tokens = new Int32Array(PROMPT_LENGTH);
-    let w = 0;
-    for (let cam = 0; cam < numCameras; cam++) {
-        for (let k = 0; k < MERGED_TOKENS_PER_IMAGE && w < tokens.length; k++) {
-            tokens[w++] = IMAGE_TOKEN_ID;
+    // Inputs are sized off the hparams, which follow the ACTIVE embodiment
+    // (numCameras, actionDim, ...) — rebuild them after every embodiment switch.
+    async function runInference(hp) {
+        const patchElems = hp.imagePatchElems;
+        if (hp.imageInputMode !== 'patches' || patchElems == null) {
+            throw new Error(`expected a patch-input model (imageInputMode 'patches' + imagePatchElems); ` +
+                `got imageInputMode=${hp.imageInputMode}`);
+        }
+        const numCameras = hp.numCameras ?? 2;
+        // Patch-input model: each camera is a pre-patchified buffer of exactly
+        // `imagePatchElems` floats. A real consumer patchifies each camera frame the
+        // way `Gr00tPolicy` does (the model's learned patch-embed runs inside the
+        // addon); we use small synthetic values.
+        const images = Array.from({ length: numCameras }, () => new Float32Array(patchElems).fill(0.02));
+        // Continuous-state model: pad the robot state to `maxStateDim`.
+        const state = vlaPadState([0, 0, 0, 0, 0, 0], hp.maxStateDim);
+        // GR00T requires the noise prior (flow-matching; it is not sampled in-model).
+        const noise = new Float32Array(hp.chunkSize * hp.maxActionDim);
+        // Prompt: one run of `MERGED_TOKENS_PER_IMAGE` image tokens per camera,
+        // each followed by a text separator, plus a short text tail. Sized off the
+        // active embodiment's camera count — a fixed 2-camera length would
+        // silently truncate the image-token runs of cameras 3+ after a switch to
+        // e.g. the 4-camera DROID row.
+        const promptLength = numCameras * (MERGED_TOKENS_PER_IMAGE + 1) + PROMPT_TEXT_TAIL;
+        const tokens = new Int32Array(promptLength);
+        let w = 0;
+        for (let cam = 0; cam < numCameras; cam++) {
+            for (let k = 0; k < MERGED_TOKENS_PER_IMAGE && w < tokens.length; k++) {
+                tokens[w++] = IMAGE_TOKEN_ID;
+            }
+            if (w < tokens.length)
+                tokens[w++] = TEXT_TOKEN_ID + cam;
+        }
+        for (; w < tokens.length; w++)
+            tokens[w] = TEXT_TOKEN_ID + w;
+        const mask = new Uint8Array(promptLength).fill(1);
+        console.log('▸ Running VLA inference...');
+        const { actions, actionDim, chunkSize, stats } = await vla({
+            modelId,
+            images,
+            // Patch inputs ignore imgWidth/imgHeight, but the request schema requires
+            // positive integers; pass the model's vision image size.
+            imgWidth: hp.visionImageSize,
+            imgHeight: hp.visionImageSize,
+            state,
+            tokens,
+            mask,
+            noise
+        });
+        console.log(`▸ Got ${chunkSize} action steps of dim ${actionDim}.`);
+        console.log(Array.from(actions.subarray(0, actionDim)));
+        if (stats) {
+            console.log(`▸ Timing: vision=${stats.vision_ms?.toFixed(0)}ms ` +
+                `ode=${stats.ode_ms?.toFixed(0)}ms ` +
+                `total=${stats.total_ms?.toFixed(0)}ms`);
         }
-        if (w < tokens.length)
-            tokens[w++] = TEXT_TOKEN_ID + cam;
     }
-    for (; w < tokens.length; w++)
-        tokens[w] = TEXT_TOKEN_ID + w;
-    const mask = new Uint8Array(PROMPT_LENGTH).fill(1);
-    console.log('▸ Running VLA inference...');
-    const { actions, actionDim, chunkSize, stats } = await vla({
-        modelId,
-        images,
-        // Patch inputs ignore imgWidth/imgHeight, but the request schema requires
-        // positive integers; pass the model's vision image size.
-        imgWidth: hparams.visionImageSize,
-        imgHeight: hparams.visionImageSize,
-        state,
-        tokens,
-        mask,
-        noise
-    });
-    console.log(`▸ Got ${chunkSize} action steps of dim ${actionDim}.`);
-    console.log(Array.from(actions.subarray(0, actionDim)));
-    if (stats) {
-        console.log(`▸ Timing: vision=${stats.vision_ms?.toFixed(0)}ms ` +
-            `ode=${stats.ode_ms?.toFixed(0)}ms ` +
-            `total=${stats.total_ms?.toFixed(0)}ms`);
+    await runInference(hparams);
+    if (switchEmbodiment !== undefined) {
+        console.log(`▸ Switching embodiment to ${JSON.stringify(switchEmbodiment)}...`);
+        const { hparams: refreshed } = await vlaSetEmbodiment({ modelId, embodiment: switchEmbodiment });
+        console.log(`▸ Embodiment: ${refreshed.selectedEmbodimentTag} ` +
+            `(cat_id ${refreshed.selectedEmbodimentCatId}, ${refreshed.numCameras} cameras)`);
+        await runInference(refreshed);
     }
     await unloadModel({ modelId, clearStorage: false });
     console.log('▸ Model unloaded.');

package/dist/index.d.ts

@@ -1,7 +1,7 @@
-export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, classify, video, type VideoProgressTick, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, audioGen, type FinetuneHandle } from './client/api/index.js';
+export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, classify, video, type VideoProgressTick, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, audioGen, type FinetuneHandle } from './client/api/index.js';
 export { close } from './client/index.js';
 export { plugins } from './client/plugins-factory.js';
-export { type LifecycleState, type ModelProgressUpdate, type LoadModelOptions, type LoadCustomPluginModelOptions, type DownloadAssetOptions, type Tool, type ToolCall, type ToolCallWithCall, type ToolCallError, type ToolCallEvent, type CompletionEvent, type CompletionFinal, type CompletionRun, type CompletionStats, type BatchCompletionEvent, type BatchCompletionResult, type BatchCompletionRun, type BatchPrompt, type EmbedStats, VERBOSITY, type Attachment, type TranscribeStreamSession, type TranscribeStreamMetadataSession, type TranscribeStreamConversationSession, type TranscribeStreamEvent, type VadStateEvent, type EndOfTurnEvent, type TranscribeSegment, type BciConfig, type BciTranscribeClientParams, type BciTranscribeStreamClientParams, type BciTranscribeStreamSession, type BciTranscribeStreamMetadataSession, type BciStreamOpts, type NeuralInput, type TextToSpeechStreamSession, type TextToSpeechStreamResponse, type TextToSpeechStreamClientParams, type TtsClientParamsInput, type TtsParlerEmotion, type TtsParlerLoadConfig, type TtsParlerRuntimeConfig, type CompletionParams, type ToolDialect, type RagSearchResult, type RagSaveEmbeddingsResult, type RagReindexResult, type RagEmbeddedDoc, type RagDoc, type RagWorkspaceInfo, type RagCloseWorkspaceParams, type RagDeleteWorkspaceParams, type RagIngestStage, type RagReindexStage, type RagSaveStage, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, type QvacConfig, type ModelInfo, type GetModelInfoParams, type GetLoadedModelInfoParams, type LoadedModelInfo, type GetSystemResourcesInput, type ResourceScope, type ResourceProvenance, type ResourceMetric, type GraphicsDriver, type GraphicsDriverCapabilities, type BackendProbeResult, type BackendDevice, type BackendDriver, type BackendFallback, type InferenceBackendDiagnostics, type CPUResourceCapabilities, type GPUResourceCapabilities, type SystemResourceCapabilities, type GPUResourceSample, type SystemResourceSample, type SystemResources, type LoadedInstance, type CacheFileInfo, toolSchema, TOOLS_MODE, type ToolsMode, type McpClient, type McpClientInput, type OCRClientParams, type OCRTextBlock, type OCROptions, type ClassifyClientParams, type ClassificationResult, type DiffusionClientParams, type DiffusionStreamResponse, type DiffusionStats, type AudioGenClientParams, type AudioGenConfig, type AudioGenRuntimeConfig, type AudioGenProgress, type AudioGenAudio, type AudioGenStats, type AudioGenResult, type AudioGenStreamResponse, type VideoClientParams, type VideoStreamResponse, type VideoStats, type UpscaleClientParams, type UpscaleStreamResponse, type UpscaleStats, type VlaConfig, type VlaClientRunParams, type VlaClientRunResult, type VlaHparams, type VlaStats, definePlugin, defineHandler, defineDuplexHandler, type QvacPlugin, type CreateModelParams, type PluginModelResult, type ModelRegistryEntry, type ModelRegistryEntryAddon, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_AUDIOGEN, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS, type BuiltinPlugin, type ProfilerMode, type FinetuneValidation, type FinetuneRunParams, type FinetuneGetStateParams, type FinetuneStopParams, type FinetuneParams, type FinetuneStatus, type FinetuneProgress, type FinetuneStats, type FinetuneResult } from './schemas/index.js';
+export { type LifecycleState, type ModelProgressUpdate, type LoadModelOptions, type LoadCustomPluginModelOptions, type DownloadAssetOptions, type Tool, type ToolCall, type ToolCallWithCall, type ToolCallError, type ToolCallEvent, type CompletionEvent, type CompletionFinal, type CompletionRun, type CompletionStats, type BatchCompletionEvent, type BatchCompletionResult, type BatchCompletionRun, type BatchPrompt, type EmbedStats, type TranscribeStats, VERBOSITY, type Attachment, type TranscribeStreamSession, type TranscribeStreamMetadataSession, type TranscribeStreamConversationSession, type TranscribeStreamEvent, type VadStateEvent, type EndOfTurnEvent, type TranscribeSegment, type BciConfig, type BciTranscribeClientParams, type BciTranscribeStreamClientParams, type BciTranscribeStreamSession, type BciTranscribeStreamMetadataSession, type BciStreamOpts, type NeuralInput, type TextToSpeechStreamSession, type TextToSpeechStreamResponse, type TextToSpeechStreamClientParams, type TtsClientParamsInput, type TtsParlerEmotion, type TtsParlerLoadConfig, type TtsParlerRuntimeConfig, TTS_PACES, TTS_COSYVOICE3_EMOTIONS, TTS_COSYVOICE3_INSTRUCT_DIALECTS, TTS_COSYVOICE3_INSTRUCT_VOLUMES, TTS_COSYVOICE3_INSTRUCT_STYLES, type TtsPace, type TtsCosyvoice3Emotion, type TtsCosyvoice3Instruct, type TtsCosyvoice3LoadConfig, type TtsCosyvoice3RuntimeConfig, type TtsAudio8LoadConfig, type TtsAudio8RuntimeConfig, type CompletionParams, type ToolDialect, type RagSearchResult, type RagSaveEmbeddingsResult, type RagReindexResult, type RagEmbeddedDoc, type RagDoc, type RagWorkspaceInfo, type RagCloseWorkspaceParams, type RagDeleteWorkspaceParams, type RagIngestStage, type RagReindexStage, type RagSaveStage, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, type QvacConfig, type ModelInfo, type GetModelInfoParams, type GetLoadedModelInfoParams, type LoadedModelInfo, type GetSystemResourcesInput, type ResourceScope, type ResourceProvenance, type ResourceMetric, type GraphicsDriver, type GraphicsDriverCapabilities, type BackendProbeResult, type BackendDevice, type BackendDriver, type BackendFallback, type InferenceBackendDiagnostics, type CPUResourceCapabilities, type GPUResourceCapabilities, type SystemResourceCapabilities, type GPUResourceSample, type SystemResourceSample, type SystemResources, type LoadedInstance, type CacheFileInfo, toolSchema, type McpClient, type McpClientInput, type OCRClientParams, type OCRTextBlock, type OCROptions, type ClassifyClientParams, type ClassificationResult, type DiffusionClientParams, type DiffusionStreamResponse, type DiffusionStats, AUDIOGEN_TASK_TYPES, AUDIOGEN_INPUT_SAMPLE_RATE, AUDIOGEN_INPUT_CHANNELS, AUDIOGEN_INPUT_MAX_SECONDS, type AudioGenTaskType, type AudioGenAudioInput, type AudioGenClientParams, type AudioGenConfig, type AudioGenRuntimeConfig, type AudioGenProgress, type AudioGenAudio, type AudioGenStats, type AudioGenResult, type AudioGenStreamResponse, type VideoClientParams, type VideoStreamResponse, type VideoStats, type UpscaleClientParams, type UpscaleStreamResponse, type UpscaleStats, type VlaConfig, type VlaClientRunParams, type VlaClientRunResult, type VlaEmbodimentSelection, type VlaEmbodimentSelector, type VlaHparams, type VlaStats, definePlugin, defineHandler, defineDuplexHandler, type QvacPlugin, type CreateModelParams, type PluginModelResult, type ModelRegistryEntry, type ModelRegistryEntryAddon, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_AUDIOGEN, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS, type BuiltinPlugin, type ProfilerMode, type FinetuneValidation, type FinetuneRunParams, type FinetuneGetStateParams, type FinetuneStopParams, type FinetuneParams, type FinetuneStatus, type FinetuneProgress, type FinetuneStats, type FinetuneResult } from './schemas/index.js';
 export { type ToolInput, type ToolHandler } from './utils/tool-helpers.js';
 export { MODEL_TYPES, ModelType } from './schemas/index.js';
 export * from './models/registry/index.js';

package/dist/index.js

@@ -1,8 +1,8 @@
 // Public API exports only
-export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, classify, video, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, audioGen } from './client/api/index.js';
+export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, classify, video, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, audioGen } from './client/api/index.js';
 export { close } from './client/index.js';
 export { plugins } from './client/plugins-factory.js';
-export { VERBOSITY, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, toolSchema, TOOLS_MODE, definePlugin, defineHandler, defineDuplexHandler, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_AUDIOGEN, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS } from './schemas/index.js';
+export { VERBOSITY, TTS_PACES, TTS_COSYVOICE3_EMOTIONS, TTS_COSYVOICE3_INSTRUCT_DIALECTS, TTS_COSYVOICE3_INSTRUCT_VOLUMES, TTS_COSYVOICE3_INSTRUCT_STYLES, SDK_CLIENT_ERROR_CODES, SDK_SERVER_ERROR_CODES, RAG_ERROR_CODES, toolSchema, AUDIOGEN_TASK_TYPES, AUDIOGEN_INPUT_SAMPLE_RATE, AUDIOGEN_INPUT_CHANNELS, AUDIOGEN_INPUT_MAX_SECONDS, definePlugin, defineHandler, defineDuplexHandler, PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_AUDIOGEN, PLUGIN_VLA, PLUGIN_CLASSIFICATION, SDK_DEFAULT_PLUGINS } from './schemas/index.js';
 export {} from './utils/tool-helpers.js';
 // Model types - canonical naming with backward-compatible aliases
 export { MODEL_TYPES, ModelType } from './schemas/index.js';