@qvac/sdk 0.15.0 → 0.16.0

package/dist/examples/diffusion-txt2vid-ltx.js

@@ -0,0 +1,92 @@
+import { loadModel, unloadModel, video } from '@qvac/sdk';
+import fs from 'fs';
+import path from 'path';
+// Text-to-video (+ synchronized audio) with LTX-2. Unlike Wan, LTX-2 uses a
+// Gemma text encoder (llmModelSrc), a video VAE (vaeModelSrc), a set of
+// text-embedding connectors (embeddingsConnectorsModelSrc — this is what
+// selects the LTX-2 layout), and an optional audio VAE (audioVaeModelSrc)
+// that produces a 48 kHz audio track muxed into the output AVI.
+//
+// This example needs powerful hardware: LTX-2.3 is a ~21B model, budget at
+// least 20+ GB of video / unified memory.
+//
+// No LTX-2 model constants ship in the SDK registry, so the model sources
+// default to public HuggingFace URLs (the SDK downloads http(s) sources
+// directly). Pass your own paths / URLs via argv to override.
+const HF = 'https://huggingface.co';
+const diffusionModelSrc = process.argv[2] ||
+    `${HF}/QuantStack/LTX-2.3-GGUF/resolve/main/LTX-2.3-distilled-1.1/LTX-2.3-22B-distilled-1.1-Q5_K_M.gguf`;
+const llmModelSrc = process.argv[3] || `${HF}/unsloth/gemma-3-12b-it-GGUF/resolve/main/gemma-3-12b-it-UD-Q4_K_XL.gguf`;
+const vaeModelSrc = process.argv[4] ||
+    `${HF}/unsloth/LTX-2.3-GGUF/resolve/main/vae/ltx-2.3-22b-distilled_video_vae.safetensors`;
+const audioVaeModelSrc = process.argv[5] ||
+    `${HF}/unsloth/LTX-2.3-GGUF/resolve/main/vae/ltx-2.3-22b-distilled_audio_vae.safetensors`;
+const embeddingsConnectorsModelSrc = process.argv[6] ||
+    `${HF}/unsloth/LTX-2.3-GGUF/resolve/main/text_encoders/ltx-2.3-22b-distilled_embeddings_connectors.safetensors`;
+const prompt = process.argv[7] || 'a claymation cat playing jazz on a piano';
+const outputDir = process.argv[8] || '.';
+try {
+    console.log('▸ Loading LTX-2 model (diffusion + Gemma + video VAE + audio VAE + connectors)...');
+    const modelId = await loadModel({
+        modelSrc: diffusionModelSrc,
+        modelType: 'sdcpp-generation',
+        modelConfig: {
+            mode: 'video',
+            device: 'gpu',
+            threads: 4,
+            // Supplying embeddingsConnectorsModelSrc selects the LTX-2 layout.
+            llmModelSrc,
+            vaeModelSrc,
+            audioVaeModelSrc,
+            embeddingsConnectorsModelSrc,
+            diffusion_fa: true,
+            vae_tiling: true
+        },
+        onProgress: (p) => {
+            const mb = (n) => (n / 1e6).toFixed(1);
+            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
+            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
+            if (p.percentage >= 100)
+                process.stderr.write('\n');
+        }
+    });
+    console.log(`▸ Model loaded: ${modelId}`);
+    console.log(`\n▸ Generating video for: "${prompt}"`);
+    const { progressStream, outputs, stats } = video({
+        modelId,
+        mode: 'txt2vid',
+        prompt,
+        negative_prompt: 'blurry, low quality, static, jittery, watermark, distorted audio',
+        // LTX-2 requires width/height as multiples of 32.
+        width: 512,
+        height: 320,
+        // LTX-2 requires (8*k + 1) frames, max 257. At 24 fps: 25≈1s, 121≈5s, 241≈10s.
+        video_frames: 121,
+        fps: 24,
+        // Distilled variants run in 4-8 steps with cfg ~1.0; the full --dev model
+        // wants steps >= 20 and cfg ~7.0.
+        steps: 10,
+        cfg_scale: 1.0,
+        seed: 42,
+        // Tile the video VAE decode along the time axis to cap peak VRAM on long
+        // / HD clips (LTX-2 only).
+        temporal_tiling: true
+    });
+    for await (const { step, totalSteps } of progressStream) {
+        console.log(`▸ step ${step}/${totalSteps}`);
+    }
+    const buffers = await outputs;
+    for (let i = 0; i < buffers.length; i++) {
+        const outputPath = path.join(outputDir, `ltx_t2v_${i}.avi`);
+        fs.writeFileSync(outputPath, buffers[i]);
+        console.log(`▸ Saved ${outputPath} (play in VLC to hear the muxed audio track)`);
+    }
+    console.log('\n▸ Stats:', await stats);
+    await unloadModel({ modelId, clearStorage: false });
+    console.log('▸ Done.');
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    process.exit(1);
+}

package/dist/examples/logging-streaming.js

@@ -46,7 +46,11 @@ try {
     const result = completion({
         modelId: llmModelId,
         history: messages,
-        stream: true
+        stream: true,
+        // Bound the generation: at temp 0.7 a 1B model can ignore "to 5" and ramble
+        // past ctx_size (2048) into a context overflow. This demo only needs enough
+        // tokens to show logs streaming.
+        generationParams: { predict: 512 }
     });
     const { embedding } = await embed({
         modelId: embedModelId,

package/dist/examples/transcription/parakeet-tdt-filesystem-chunked.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=parakeet-tdt-filesystem-chunked.d.ts.map

package/dist/examples/transcription/parakeet-tdt-filesystem-chunked.js

@@ -0,0 +1,203 @@
+/**
+ * Parakeet TDT transcription for long audio files.
+ *
+ * Usage:
+ *   bun run examples/transcription/parakeet-tdt-filesystem-chunked.ts <audio-file> [parakeet-tdt-gguf]
+ *
+ * FFmpeg first detects silence, then this example selects boundaries near
+ * 45 seconds and transcribes one segment at a time. Segments never exceed
+ * 60 seconds, which keeps audio and inference memory bounded. Any audio
+ * format supported by FFmpeg can be used.
+ */
+import { loadModel, unloadModel, transcribe, PARAKEET_TDT_0_6B_V3_Q8_0 } from '@qvac/sdk';
+import { spawn } from 'child_process';
+const SAMPLE_RATE = 16000;
+const SILENCE_NOISE_DB = -35;
+const SILENCE_DURATION_S = 0.5;
+const MIN_SEGMENT_S = 10;
+const TARGET_SEGMENT_S = 45;
+const MAX_SEGMENT_S = 60;
+function runCommand(command, args) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+        const stdout = [];
+        const stderr = [];
+        child.stdout.on('data', (chunk) => stdout.push(chunk));
+        child.stderr.on('data', (chunk) => stderr.push(chunk));
+        child.on('error', reject);
+        child.on('close', (code) => {
+            const stderrText = Buffer.concat(stderr).toString();
+            if (code !== 0) {
+                reject(new Error(`${command} exited with code ${code}: ${stderrText.trim()}`));
+                return;
+            }
+            resolve({ stdout: Buffer.concat(stdout), stderr: stderrText });
+        });
+    });
+}
+async function getAudioDuration(audioPath) {
+    const result = await runCommand('ffprobe', [
+        '-v',
+        'error',
+        '-show_entries',
+        'format=duration',
+        '-of',
+        'default=noprint_wrappers=1:nokey=1',
+        audioPath
+    ]);
+    const duration = Number.parseFloat(result.stdout.toString().trim());
+    if (!Number.isFinite(duration) || duration <= 0) {
+        throw new Error(`Could not determine audio duration for ${audioPath}`);
+    }
+    return duration;
+}
+async function detectSilences(audioPath, duration) {
+    const result = await runCommand('ffmpeg', [
+        '-hide_banner',
+        '-nostats',
+        '-i',
+        audioPath,
+        '-vn',
+        '-af',
+        `silencedetect=noise=${SILENCE_NOISE_DB}dB:d=${SILENCE_DURATION_S}`,
+        '-f',
+        'null',
+        '-'
+    ]);
+    const intervals = [];
+    let currentStart = null;
+    for (const line of result.stderr.split('\n')) {
+        const startMatch = line.match(/silence_start:\s*([0-9.]+)/);
+        if (startMatch?.[1]) {
+            const start = Number.parseFloat(startMatch[1]);
+            if (Number.isFinite(start))
+                currentStart = start;
+        }
+        const endMatch = line.match(/silence_end:\s*([0-9.]+)/);
+        if (endMatch?.[1] && currentStart !== null) {
+            const end = Number.parseFloat(endMatch[1]);
+            if (Number.isFinite(end) && end > currentStart) {
+                intervals.push({ start: currentStart, end });
+            }
+            currentStart = null;
+        }
+    }
+    if (currentStart !== null && currentStart < duration) {
+        intervals.push({ start: currentStart, end: duration });
+    }
+    return intervals;
+}
+function createAudioSegments(duration, silences) {
+    const silenceMidpoints = silences.map(({ start, end }) => start + (end - start) / 2);
+    const segments = [];
+    let start = 0;
+    while (duration - start > MAX_SEGMENT_S) {
+        const target = start + TARGET_SEGMENT_S;
+        const minimumCut = start + MIN_SEGMENT_S;
+        const maximumCut = start + MAX_SEGMENT_S;
+        const candidates = silenceMidpoints.filter((candidate) => candidate >= minimumCut && candidate <= maximumCut);
+        const firstCandidate = candidates[0];
+        const cut = firstCandidate === undefined
+            ? Math.min(target, duration - MIN_SEGMENT_S)
+            : candidates.slice(1).reduce((best, candidate) => {
+                return Math.abs(candidate - target) < Math.abs(best - target) ? candidate : best;
+            }, firstCandidate);
+        segments.push({ start, end: cut });
+        start = cut;
+    }
+    if (duration > start) {
+        segments.push({ start, end: duration });
+    }
+    return segments;
+}
+async function decodeSegment(audioPath, segment) {
+    const result = await runCommand('ffmpeg', [
+        '-hide_banner',
+        '-loglevel',
+        'error',
+        '-ss',
+        segment.start.toFixed(3),
+        '-i',
+        audioPath,
+        '-t',
+        (segment.end - segment.start).toFixed(3),
+        '-vn',
+        '-ar',
+        String(SAMPLE_RATE),
+        '-ac',
+        '1',
+        '-sample_fmt',
+        's16',
+        '-f',
+        's16le',
+        'pipe:1'
+    ]);
+    return result.stdout;
+}
+function formatTimestamp(seconds) {
+    const totalTenths = Math.round(seconds * 10);
+    const hours = Math.floor(totalTenths / 36000);
+    const minutes = Math.floor((totalTenths % 36000) / 600);
+    const remainingSeconds = ((totalTenths % 600) / 10).toFixed(1).padStart(4, '0');
+    return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${remainingSeconds}`;
+}
+const args = process.argv.slice(2);
+if (!args[0]) {
+    console.error('Usage: bun run examples/transcription/parakeet-tdt-filesystem-chunked.ts ' +
+        '<audio-file> [parakeet-tdt-gguf]');
+    process.exit(1);
+}
+const audioFilePath = args[0];
+const parakeetModelSrc = args[1] ?? PARAKEET_TDT_0_6B_V3_Q8_0;
+let modelId = null;
+try {
+    await runCommand('ffmpeg', ['-version']);
+    await runCommand('ffprobe', ['-version']);
+    console.log('▸ Detecting silence and planning bounded segments...');
+    const duration = await getAudioDuration(audioFilePath);
+    const silences = await detectSilences(audioFilePath, duration);
+    const segments = createAudioSegments(duration, silences);
+    console.log(`▸ Audio duration: ${formatTimestamp(duration)}; detected ${silences.length} silence interval(s); planned ${segments.length} segment(s)`);
+    console.log('▸ Loading Parakeet TDT model...');
+    modelId = await loadModel({
+        modelSrc: parakeetModelSrc,
+        modelType: 'parakeet-transcription',
+        onProgress: (p) => {
+            const mb = (n) => (n / 1e6).toFixed(1);
+            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
+            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
+            if (p.percentage >= 100)
+                process.stderr.write('\n');
+        }
+    });
+    console.log(`▸ Model loaded: ${modelId}`);
+    const transcript = [];
+    for (const [index, segment] of segments.entries()) {
+        console.log(`▸ Segment ${index + 1}/${segments.length}: ${formatTimestamp(segment.start)} - ${formatTimestamp(segment.end)}`);
+        const audioChunk = await decodeSegment(audioFilePath, segment);
+        const text = await transcribe({ modelId, audioChunk });
+        const normalizedText = text.trim();
+        if (normalizedText && !normalizedText.includes('[No speech detected]')) {
+            transcript.push(normalizedText);
+            console.log(normalizedText);
+        }
+    }
+    console.log('\n▸ Complete transcript');
+    console.log(transcript.join(' '));
+    await unloadModel({ modelId });
+    modelId = null;
+    console.log('▸ Done');
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    if (modelId) {
+        try {
+            await unloadModel({ modelId });
+        }
+        catch (unloadError) {
+            console.error('✖ Failed to unload model:', unloadError);
+        }
+    }
+    process.exit(1);
+}

package/dist/examples/vla-groot.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=vla-groot.d.ts.map

package/dist/examples/vla-groot.js

@@ -0,0 +1,118 @@
+/**
+ * NVIDIA GR00T N1.7-3B (vision-language-action) example using the QVAC SDK.
+ *
+ * Loads the GR00T-N1.7-3B LIBERO GGUF model, runs a single inference pass with
+ * synthetic inputs, and prints the produced action chunk + per-stage timings.
+ *
+ * GR00T differs from SmolVLA / π₀.₅ in the ways the SDK surfaces via
+ * `vlaHparams()`:
+ *   - `imageInputMode: 'patches'` — each `images` entry is a pre-patchified
+ *     buffer of `hparams.imagePatchElems` floats, NOT a `3·w·h` pixel plane.
+ *     A real consumer runs the model's own vision patch-embed over each camera
+ *     frame; here we feed synthetic patch buffers of the right length.
+ *   - `stateInputMode: 'continuous'` — the robot state is projected by an
+ *     in-model linear layer; pad it to `hparams.maxStateDim` with `vlaPadState`.
+ *   - `noise` is REQUIRED. GR00T is a flow-matching model that does not sample
+ *     its own prior; a missing `noise` is rejected as INVALID_INPUT.
+ *
+ * The prompt must place one contiguous run of `MERGED_TOKENS_PER_IMAGE`
+ * image-placeholder tokens per camera. A real consumer produces this by
+ * tokenising the instruction with the Qwen3-VL tokenizer, which inserts the
+ * image tokens; we lay it out by hand for the smoke.
+ *
+ * Usage:
+ *   bun examples/vla-groot.ts [path-to-groot.gguf]
+ *
+ * By default the example pulls the registry-baked GR00T-LIBERO GGUF (~3.76 GB)
+ * on first run and caches it locally. Pass an absolute path on the command line
+ * to override and load a local GGUF instead.
+ */
+import { close, loadModel, GROOT_Q8_VF16, unloadModel, vla, vlaHparams, vlaPadState } from '@qvac/sdk';
+// LIBERO GR00T prompt layout (Qwen3-VL tokenizer). GR00T reports
+// `hparams.tokenizerMaxLength === 0` — it does not surface a tokenizer length,
+// so the prompt length is fixed by the model: 2 cameras × 64 merged image
+// tokens (256 patches, 2×2 merge) = 128 image tokens, plus text, = 148 total.
+// A real consumer gets these from the tokenizer; the smoke hard-codes them.
+const IMAGE_TOKEN_ID = 151655;
+const MERGED_TOKENS_PER_IMAGE = 64;
+const PROMPT_LENGTH = 148;
+const TEXT_TOKEN_ID = 1000;
+const modelSrcOverride = process.argv[2];
+const modelSrc = modelSrcOverride ?? GROOT_Q8_VF16;
+try {
+    console.log('▸ Loading GR00T (N1.7-3B LIBERO) model...');
+    const modelId = await loadModel({
+        modelSrc,
+        modelType: 'ggml-vla',
+        modelConfig: { backend: 'cpu' },
+        onProgress: (p) => {
+            const mb = (n) => (n / 1e6).toFixed(1);
+            const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
+            process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
+            if (p.percentage >= 100)
+                process.stderr.write('\n');
+        }
+    });
+    if (typeof modelSrc !== 'string')
+        process.stderr.write('\n');
+    console.log(`▸ Model loaded: ${modelId}`);
+    const { hparams, backendName } = await vlaHparams({ modelId });
+    console.log(`▸ Backend: ${backendName ?? '(unknown)'}`);
+    console.log('▸ Hparams:', hparams);
+    const patchElems = hparams.imagePatchElems;
+    if (hparams.imageInputMode !== 'patches' || patchElems == null) {
+        throw new Error(`expected a patch-input model (imageInputMode 'patches' + imagePatchElems); ` +
+            `got imageInputMode=${hparams.imageInputMode}`);
+    }
+    const numCameras = hparams.numCameras ?? 2;
+    // Patch-input model: each camera is a pre-patchified buffer of exactly
+    // `imagePatchElems` floats. A real consumer fills these from the model's
+    // vision patch-embed over the camera frame; we use small synthetic values.
+    const images = Array.from({ length: numCameras }, () => new Float32Array(patchElems).fill(0.02));
+    // Continuous-state model: pad the robot state to `maxStateDim`.
+    const state = vlaPadState([0, 0, 0, 0, 0, 0], hparams.maxStateDim);
+    // GR00T requires the noise prior (flow-matching; it is not sampled in-model).
+    const noise = new Float32Array(hparams.chunkSize * hparams.maxActionDim);
+    // Prompt: one run of `MERGED_TOKENS_PER_IMAGE` image tokens per camera, each
+    // followed by a text separator, padded out to the fixed `PROMPT_LENGTH`.
+    const tokens = new Int32Array(PROMPT_LENGTH);
+    let w = 0;
+    for (let cam = 0; cam < numCameras; cam++) {
+        for (let k = 0; k < MERGED_TOKENS_PER_IMAGE && w < tokens.length; k++) {
+            tokens[w++] = IMAGE_TOKEN_ID;
+        }
+        if (w < tokens.length)
+            tokens[w++] = TEXT_TOKEN_ID + cam;
+    }
+    for (; w < tokens.length; w++)
+        tokens[w] = TEXT_TOKEN_ID + w;
+    const mask = new Uint8Array(PROMPT_LENGTH).fill(1);
+    console.log('▸ Running VLA inference...');
+    const { actions, actionDim, chunkSize, stats } = await vla({
+        modelId,
+        images,
+        // Patch inputs ignore imgWidth/imgHeight, but the request schema requires
+        // positive integers; pass the model's vision image size.
+        imgWidth: hparams.visionImageSize,
+        imgHeight: hparams.visionImageSize,
+        state,
+        tokens,
+        mask,
+        noise
+    });
+    console.log(`▸ Got ${chunkSize} action steps of dim ${actionDim}.`);
+    console.log(Array.from(actions.subarray(0, actionDim)));
+    if (stats) {
+        console.log(`▸ Timing: vision=${stats.vision_ms?.toFixed(0)}ms ` +
+            `ode=${stats.ode_ms?.toFixed(0)}ms ` +
+            `total=${stats.total_ms?.toFixed(0)}ms`);
+    }
+    await unloadModel({ modelId, clearStorage: false });
+    console.log('▸ Model unloaded.');
+    process.exit(0);
+}
+catch (error) {
+    console.error('✖', error);
+    await close();
+    process.exit(1);
+}