@qvac/sdk 0.18.2 → 0.19.0

package/dist/client/api/video.d.ts

@@ -1,87 +0,0 @@
-import { type VideoClientParams, type VideoStats } from '../../schemas/index.js';
-export interface VideoProgressTick {
-    step: number;
-    totalSteps: number;
-    elapsedMs: number;
-}
-export interface VideoResult {
-    requestId: string;
-    progressStream: AsyncGenerator<VideoProgressTick>;
-    outputs: Promise<Uint8Array[]>;
-    stats: Promise<VideoStats | undefined>;
-}
-/**
- * Generates a video using a loaded video diffusion model.
- *
- * @param params - Video request parameters (model, prompt, dimensions, frame count, fps, sampler, seed, etc.).
- * @returns A result object exposing `requestId` (stable identifier for this in-flight generation), `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated video buffers, typically a single AVI file), and `stats` (promise of generation statistics).
- *
- * Supports `txt2vid` (text-to-video) and `img2vid` (image-to-video) for both the
- * Wan and LTX-2 layouts. For Wan `img2vid`, load the pipeline with
- * `modelConfig.clipVisionModelSrc` set to `clip_vision_h.safetensors`; LTX-2
- * `img2vid` conditions on the first frame through its video VAE and needs no
- * CLIP vision weights (the same LTX-2 model loaded for txt2vid also does
- * img2vid). On React Native, prefer a `modelId` loaded with a `delegate` since
- * the bundled video diffusion models are too large for typical mobile devices.
- *
- * @example Basic txt2vid generation
- * ```typescript
- * const { outputs, stats } = video({
- *   modelId,
- *   mode: "txt2vid",
- *   prompt: "a cat surfing a wave at sunset",
- *   width: 480,
- *   height: 832,
- *   video_frames: 17, // must satisfy (4*k + 1)
- *   fps: 16,
- * });
- * const buffers = await outputs;
- * fs.writeFileSync("output.avi", buffers[0]);
- * ```
- *
- * @example With progress tracking
- * ```typescript
- * const { progressStream, outputs } = video({
- *   modelId,
- *   mode: "txt2vid",
- *   prompt: "a sunset over the ocean",
- * });
- * for await (const { step, totalSteps } of progressStream) {
- *   console.log(`${step}/${totalSteps}`);
- * }
- * const buffers = await outputs;
- * ```
- *
- * @example Image-to-video (first frame + motion prompt)
- * ```typescript
- * const firstFrame = fs.readFileSync("portrait.png");
- * const { outputs } = video({
- *   modelId,
- *   mode: "img2vid",
- *   prompt: "the subject slowly turns and smiles, cinematic lighting",
- *   init_image: firstFrame,
- *   strength: 0.85,
- * });
- * ```
- *
- * @example With control frames (e.g. for guided generation)
- * ```typescript
- * const frameA = fs.readFileSync("frame-a.png");
- * const frameB = fs.readFileSync("frame-b.png");
- * const { outputs } = video({
- *   modelId,
- *   mode: "txt2vid",
- *   prompt: "smooth transition between scenes",
- *   control_frames: [frameA, frameB],
- * });
- * ```
- *
- * @example Cancellation via requestId
- * ```typescript
- * const { requestId, outputs } = video({ modelId, mode: "txt2vid", prompt: "..." });
- * // ...later
- * await cancel(requestId);
- * ```
- */
-export declare function video(params: VideoClientParams): VideoResult;
-//# sourceMappingURL=video.d.ts.map

package/dist/client/api/video.js

@@ -1,175 +0,0 @@
-import { videoStreamResponseSchema } from '../../schemas/index.js';
-import { stream as streamRpc } from '../../client/rpc/rpc-client.js';
-import { generateClientRequestId } from '../../client/api/client-request-id.js';
-import { decodeBase64, encodeBase64 } from '../../utils/encoding.js';
-/**
- * Generates a video using a loaded video diffusion model.
- *
- * @param params - Video request parameters (model, prompt, dimensions, frame count, fps, sampler, seed, etc.).
- * @returns A result object exposing `requestId` (stable identifier for this in-flight generation), `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated video buffers, typically a single AVI file), and `stats` (promise of generation statistics).
- *
- * Supports `txt2vid` (text-to-video) and `img2vid` (image-to-video) for both the
- * Wan and LTX-2 layouts. For Wan `img2vid`, load the pipeline with
- * `modelConfig.clipVisionModelSrc` set to `clip_vision_h.safetensors`; LTX-2
- * `img2vid` conditions on the first frame through its video VAE and needs no
- * CLIP vision weights (the same LTX-2 model loaded for txt2vid also does
- * img2vid). On React Native, prefer a `modelId` loaded with a `delegate` since
- * the bundled video diffusion models are too large for typical mobile devices.
- *
- * @example Basic txt2vid generation
- * ```typescript
- * const { outputs, stats } = video({
- *   modelId,
- *   mode: "txt2vid",
- *   prompt: "a cat surfing a wave at sunset",
- *   width: 480,
- *   height: 832,
- *   video_frames: 17, // must satisfy (4*k + 1)
- *   fps: 16,
- * });
- * const buffers = await outputs;
- * fs.writeFileSync("output.avi", buffers[0]);
- * ```
- *
- * @example With progress tracking
- * ```typescript
- * const { progressStream, outputs } = video({
- *   modelId,
- *   mode: "txt2vid",
- *   prompt: "a sunset over the ocean",
- * });
- * for await (const { step, totalSteps } of progressStream) {
- *   console.log(`${step}/${totalSteps}`);
- * }
- * const buffers = await outputs;
- * ```
- *
- * @example Image-to-video (first frame + motion prompt)
- * ```typescript
- * const firstFrame = fs.readFileSync("portrait.png");
- * const { outputs } = video({
- *   modelId,
- *   mode: "img2vid",
- *   prompt: "the subject slowly turns and smiles, cinematic lighting",
- *   init_image: firstFrame,
- *   strength: 0.85,
- * });
- * ```
- *
- * @example With control frames (e.g. for guided generation)
- * ```typescript
- * const frameA = fs.readFileSync("frame-a.png");
- * const frameB = fs.readFileSync("frame-b.png");
- * const { outputs } = video({
- *   modelId,
- *   mode: "txt2vid",
- *   prompt: "smooth transition between scenes",
- *   control_frames: [frameA, frameB],
- * });
- * ```
- *
- * @example Cancellation via requestId
- * ```typescript
- * const { requestId, outputs } = video({ modelId, mode: "txt2vid", prompt: "..." });
- * // ...later
- * await cancel(requestId);
- * ```
- */
-export function video(params) {
-    const requestId = generateClientRequestId();
-    const { control_frames, init_image, ...rest } = params;
-    const request = {
-        ...rest,
-        ...(control_frames !== undefined && {
-            control_frames: control_frames.map(encodeBase64)
-        }),
-        ...(init_image !== undefined && {
-            init_image: encodeBase64(init_image)
-        }),
-        type: 'videoStream',
-        requestId
-    };
-    let statsResolver = () => { };
-    let statsRejecter = () => { };
-    const statsPromise = new Promise((resolve, reject) => {
-        statsResolver = resolve;
-        statsRejecter = reject;
-    });
-    statsPromise.catch(() => { });
-    const progressQueue = [];
-    const collectedBuffers = [];
-    let progressDone = false;
-    let progressResolve = null;
-    let streamError = null;
-    let outputsResolver = () => { };
-    let outputsRejecter = () => { };
-    const outputsPromise = new Promise((resolve, reject) => {
-        outputsResolver = resolve;
-        outputsRejecter = reject;
-    });
-    outputsPromise.catch(() => { });
-    async function processResponses() {
-        try {
-            for await (const response of streamRpc(request)) {
-                if (response &&
-                    typeof response === 'object' &&
-                    'type' in response &&
-                    response.type === 'videoStream') {
-                    const parsed = videoStreamResponseSchema.parse(response);
-                    if (parsed.step != null && parsed.totalSteps != null && parsed.elapsedMs != null) {
-                        progressQueue.push({
-                            step: parsed.step,
-                            totalSteps: parsed.totalSteps,
-                            elapsedMs: parsed.elapsedMs
-                        });
-                        if (progressResolve) {
-                            progressResolve();
-                            progressResolve = null;
-                        }
-                    }
-                    if (parsed.data) {
-                        collectedBuffers.push(decodeBase64(parsed.data));
-                    }
-                    if (parsed.done) {
-                        statsResolver(parsed.stats);
-                        outputsResolver(collectedBuffers);
-                    }
-                }
-            }
-        }
-        catch (error) {
-            streamError = error instanceof Error ? error : new Error(String(error));
-            statsRejecter(streamError);
-            outputsRejecter(streamError);
-        }
-        progressDone = true;
-        if (progressResolve) {
-            progressResolve();
-            progressResolve = null;
-        }
-    }
-    void processResponses();
-    const progressStream = (async function* () {
-        while (true) {
-            if (progressQueue.length > 0) {
-                yield progressQueue.shift();
-            }
-            else if (progressDone) {
-                if (streamError)
-                    throw streamError;
-                return;
-            }
-            else {
-                await new Promise((resolve) => {
-                    progressResolve = resolve;
-                });
-            }
-        }
-    })();
-    return {
-        requestId,
-        progressStream,
-        outputs: outputsPromise,
-        stats: statsPromise
-    };
-}

package/dist/client/api/vla-helpers.d.ts

@@ -1,45 +0,0 @@
-/**
- * Pure-JS helpers for preparing VLA inference inputs on the client side.
- *
- * Mirrors `preprocessImage` / `padState` / `DEFAULT_IMAGE_SIZE` from
- * `@qvac/vla-ggml/addon.js`. Kept inlined (not re-exported from the addon)
- * so the SDK's client surface never loads the addon's native binding —
- * consumers running under Node / Bun / Expo without VLA prebuilds can still
- * use the SDK to drive a remote VLA worker.
- *
- * This file is a verbatim port of the addon's JS implementation; keep them
- * in sync so the wire-format tensors stay byte-identical regardless of
- * where preprocessing runs (consumer process vs worker vs addon).
- */
-/** Default vision tower image size used by SmolVLA-LIBERO. */
-export declare const VLA_DEFAULT_IMAGE_SIZE = 512;
-type PixelsInput = Float32Array | Uint8Array | number[];
-type ImageLayout = 'hwc' | 'chw';
-export interface VlaPreprocessImageOptions {
-    size?: number;
-    layout?: ImageLayout;
-    /**
-     * Skip the [0,255] vs [0,1] auto-detection heuristic when the caller knows
-     * the range. `1` = pixels already in [0,1] (no rescale). `1/255` = pixels
-     * in [0,255] (rescale to [0,1]). Any other value (including the literal
-     * `'auto'`) falls back to the heuristic.
-     */
-    scale?: number | 'auto';
-}
-/**
- * Resize + letterbox + normalize a camera frame to `(3, size, size)` Float32
- * in `[-1, 1]`. Drop-in equivalent of `@qvac/vla-ggml`'s `preprocessImage`.
- *
- * Letterbox places the resized content at the **bottom-right** with padding
- * at top/left (`padLeft = size - newW`, `padTop = size - newH`), matching
- * the reference smolvla.cpp behavior.
- */
-export declare function vlaPreprocessImage(pixels: PixelsInput, width: number, height: number, opts?: VlaPreprocessImageOptions): Float32Array;
-/**
- * Zero-pad a state vector to `targetDim`. Extra entries are zero-initialised;
- * input longer than `targetDim` raises. Mirrors how smolvla.cpp expects the
- * state tensor (`max_state_dim` = 32 by default).
- */
-export declare function vlaPadState(state: ArrayLike<number>, targetDim?: number): Float32Array;
-export {};
-//# sourceMappingURL=vla-helpers.d.ts.map

package/dist/client/api/vla-helpers.js

@@ -1,125 +0,0 @@
-/**
- * Pure-JS helpers for preparing VLA inference inputs on the client side.
- *
- * Mirrors `preprocessImage` / `padState` / `DEFAULT_IMAGE_SIZE` from
- * `@qvac/vla-ggml/addon.js`. Kept inlined (not re-exported from the addon)
- * so the SDK's client surface never loads the addon's native binding —
- * consumers running under Node / Bun / Expo without VLA prebuilds can still
- * use the SDK to drive a remote VLA worker.
- *
- * This file is a verbatim port of the addon's JS implementation; keep them
- * in sync so the wire-format tensors stay byte-identical regardless of
- * where preprocessing runs (consumer process vs worker vs addon).
- */
-/** Default vision tower image size used by SmolVLA-LIBERO. */
-export const VLA_DEFAULT_IMAGE_SIZE = 512;
-function detectScale(pixels) {
-    if (pixels instanceof Uint8Array)
-        return 1 / 255;
-    // Float/Number arrays: scan a small window to decide whether it's [0,255] or [0,1].
-    const limit = Math.min(pixels.length, 256);
-    let maxVal = 0;
-    for (let i = 0; i < limit; i++) {
-        const v = pixels[i];
-        if (v > maxVal)
-            maxVal = v;
-    }
-    return maxVal > 1.001 ? 1 / 255 : 1;
-}
-/**
- * Resize + letterbox + normalize a camera frame to `(3, size, size)` Float32
- * in `[-1, 1]`. Drop-in equivalent of `@qvac/vla-ggml`'s `preprocessImage`.
- *
- * Letterbox places the resized content at the **bottom-right** with padding
- * at top/left (`padLeft = size - newW`, `padTop = size - newH`), matching
- * the reference smolvla.cpp behavior.
- */
-export function vlaPreprocessImage(pixels, width, height, opts = {}) {
-    const size = opts.size ?? VLA_DEFAULT_IMAGE_SIZE;
-    const layout = opts.layout ?? 'hwc';
-    if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
-        throw new TypeError('vlaPreprocessImage: width/height must be positive integers');
-    }
-    const expected = width * height * 3;
-    if (pixels.length !== expected) {
-        throw new RangeError(`vlaPreprocessImage: expected ${expected} pixel values, got ${pixels.length}`);
-    }
-    const normalize = opts.scale === 1 || opts.scale === 1 / 255 ? opts.scale : detectScale(pixels);
-    // Letterbox target size (aspect-ratio preserving).
-    const ratio = Math.max(width / size, height / size);
-    const newW = Math.max(1, Math.floor(width / ratio));
-    const newH = Math.max(1, Math.floor(height / ratio));
-    const padLeft = size - newW;
-    const padTop = size - newH;
-    const xScale = width / newW;
-    const yScale = height / newH;
-    // Output starts at -1 so the pad region is already in [-1, 1] and we only
-    // need to overwrite the (newH × newW) inner region with the resized content.
-    const out = new Float32Array(3 * size * size);
-    out.fill(-1);
-    const planeStride = size * size;
-    const widthHeight = width * height;
-    for (let yy = 0; yy < newH; yy++) {
-        const yIn = (yy + 0.5) * yScale - 0.5;
-        const y0 = Math.max(0, Math.floor(yIn));
-        const y1 = Math.min(height - 1, y0 + 1);
-        const dy = Math.min(1, Math.max(0, yIn - y0));
-        const dyInv = 1 - dy;
-        const outY = yy + padTop;
-        for (let xx = 0; xx < newW; xx++) {
-            const xIn = (xx + 0.5) * xScale - 0.5;
-            const x0 = Math.max(0, Math.floor(xIn));
-            const x1 = Math.min(width - 1, x0 + 1);
-            const dx = Math.min(1, Math.max(0, xIn - x0));
-            const dxInv = 1 - dx;
-            const outX = xx + padLeft;
-            const w00 = dxInv * dyInv;
-            const w10 = dx * dyInv;
-            const w01 = dxInv * dy;
-            const w11 = dx * dy;
-            const outIdx = outY * size + outX;
-            if (layout === 'hwc') {
-                const i00 = (y0 * width + x0) * 3;
-                const i10 = (y0 * width + x1) * 3;
-                const i01 = (y1 * width + x0) * 3;
-                const i11 = (y1 * width + x1) * 3;
-                for (let c = 0; c < 3; c++) {
-                    const v = pixels[i00 + c] * w00 +
-                        pixels[i10 + c] * w10 +
-                        pixels[i01 + c] * w01 +
-                        pixels[i11 + c] * w11;
-                    // Apply scale and shift into [-1, 1] in one fused multiply-add.
-                    out[c * planeStride + outIdx] = v * normalize * 2 - 1;
-                }
-            }
-            else {
-                for (let c = 0; c < 3; c++) {
-                    const plane = c * widthHeight;
-                    const v = pixels[plane + y0 * width + x0] * w00 +
-                        pixels[plane + y0 * width + x1] * w10 +
-                        pixels[plane + y1 * width + x0] * w01 +
-                        pixels[plane + y1 * width + x1] * w11;
-                    out[c * planeStride + outIdx] = v * normalize * 2 - 1;
-                }
-            }
-        }
-    }
-    return out;
-}
-/**
- * Zero-pad a state vector to `targetDim`. Extra entries are zero-initialised;
- * input longer than `targetDim` raises. Mirrors how smolvla.cpp expects the
- * state tensor (`max_state_dim` = 32 by default).
- */
-export function vlaPadState(state, targetDim = 32) {
-    if (!Number.isInteger(targetDim) || targetDim <= 0) {
-        throw new TypeError('vlaPadState: targetDim must be a positive integer');
-    }
-    if (state.length > targetDim) {
-        throw new RangeError(`vlaPadState: input length ${state.length} exceeds targetDim ${targetDim}`);
-    }
-    const out = new Float32Array(targetDim);
-    for (let i = 0; i < state.length; i++)
-        out[i] = state[i];
-    return out;
-}

package/dist/client/api/vla.d.ts

@@ -1,114 +0,0 @@
-import { type VlaClientRunParams, type VlaClientRunResult, type VlaEmbodimentSelection, type VlaHparams } from '../../schemas/index.js';
-/**
- * Run VLA inference on a loaded model (SmolVLA, π₀.₅, or GR00T) and return the
- * produced action chunk plus per-stage timings.
- *
- * @param params - Inference inputs.
- * @param params.modelId - Identifier of the loaded VLA model (returned by
- *   `loadModel({ modelType: "vla", ... })`).
- * @param params.images - The preprocessed camera frames, one per camera. For
- *   pixel-input models (SmolVLA, π₀.₅) each is a `Float32Array` of length
- *   `3 * imgWidth * imgHeight` in CHW layout, range `[-1, 1]`, produced by the
- *   addon's `preprocessImage()` (re-exported as `vlaPreprocessImage`). For
- *   patch-input models (GR00T, `hparams.imageInputMode === 'patches'`) each is
- *   the pre-patchified buffer of length `hparams.imagePatchElems` instead. Pass
- *   exactly `hparams.numCameras` frames (2 for SmolVLA and GR00T, 3 for π₀.₅).
- * @param params.imgWidth - Width of each preprocessed image; must equal
- *   `hparams.visionImageSize`.
- * @param params.imgHeight - Height of each preprocessed image; must equal
- *   `hparams.visionImageSize`.
- * @param params.state - Robot end-effector / gripper state. For
- *   continuous-state models (SmolVLA, GR00T) pad to `hparams.maxStateDim` with
- *   `vlaPadState`. For discrete-state models (π₀.₅,
- *   `hparams.stateInputMode === 'discrete'`) the state is tokenised into the
- *   prompt and this buffer is ignored — pass an empty `Float32Array(0)`.
- * @param params.tokens - Tokenized instruction (`Int32Array` of length
- *   `hparams.tokenizerMaxLength`). Tokenize on the consumer side with the
- *   model's tokenizer (SmolVLM2 for SmolVLA, PaliGemma/Gemma for π₀.₅,
- *   Qwen3-VL for GR00T).
- * @param params.mask - Token attention mask (`Uint8Array` matching `tokens`).
- * @param params.noise - Seeded noise prior
- *   (`Float32Array` of length `hparams.chunkSize * hparams.maxActionDim`).
- *   Optional for SmolVLA / π₀.₅ (addon samples its own prior when omitted).
- *   Required for GR00T (`hparams.imageInputMode === 'patches'`).
- * @returns A `VlaClientRunResult` with the produced `actions` Float32Array
- *   (length `chunkSize * actionDim`), the corresponding `chunkSize` /
- *   `actionDim` returned by the addon, and optional per-stage `stats`.
- *
- * @example
- * ```typescript
- * import { loadModel, vla, vlaPreprocessImage, vlaPadState, vlaHparams } from "@qvac/sdk";
- *
- * const modelId = await loadModel({ modelSrc: "/path/to/smolvla.gguf", modelType: "vla" });
- * const { hparams } = await vlaHparams({ modelId });
- * const size = hparams.visionImageSize;
- * const front = vlaPreprocessImage(frontPixels, frontW, frontH, { size });
- * const wrist = vlaPreprocessImage(wristPixels, wristW, wristH, { size });
- * const state = vlaPadState(robotState, hparams.maxStateDim);
- * const tokens = new Int32Array(hparams.tokenizerMaxLength);
- * const mask = new Uint8Array(hparams.tokenizerMaxLength);
- * // ...tokenize the instruction into tokens/mask...
- * const { actions } = await vla({
- *   modelId, images: [front, wrist], imgWidth: size, imgHeight: size,
- *   state, tokens, mask,
- * });
- * ```
- */
-export declare function vla(params: VlaClientRunParams): Promise<VlaClientRunResult>;
-/**
- * Fetch the loaded VLA model's hyperparameters and the active ggml backend
- * name. Useful to size token / state / noise buffers before calling `vla()`.
- *
- * @param params - Identifier of the loaded VLA model.
- * @returns The model's hparams and the human-readable backend name
- *   (`"CPU"` / `"Vulkan"` / `"Metal"` / `"OpenCL"` / `null` if the addon
- *   has not surfaced one).
- */
-export declare function vlaHparams(params: {
-    modelId: string;
-}): Promise<{
-    hparams: VlaHparams;
-    backendName: string | null;
-}>;
-/**
- * Switch a loaded multi-embodiment GR00T model to another embodiment shipped
- * in the same GGUF — no reload, only that embodiment's weight rows are re-read.
- *
- * The returned hparams follow the newly active embodiment: re-read
- * `numCameras` (and re-size buffers if needed) before the next `vla()` call.
- * `selectedEmbodimentTag` / `selectedEmbodimentCatId` report what was
- * resolved. Selecting by `catId` reports that id's canonical tag, which may
- * differ from the alias used to select — the id is the stable identity.
- *
- * Rejects (leaving the current embodiment active) for a non-GR00T or
- * single-embodiment model, an unknown tag or `cat_id`, one not stored in the
- * GGUF, an embodiment with no known camera count and no `numCameras`
- * override, or while an inference response is still pending — await the
- * in-flight `vla()` call first.
- *
- * @param params.modelId - Identifier of the loaded VLA model.
- * @param params.embodiment - The embodiment to activate: a tag string, a
- *   numeric `cat_id` (0..31), or `{ tag | catId, numCameras }` where
- *   `numCameras` overrides the camera count stored in the GGUF.
- * @returns The refreshed model hparams after the switch.
- *
- * @example
- * ```typescript
- * import { loadModel, vlaSetEmbodiment, vlaHparams } from "@qvac/sdk";
- *
- * const modelId = await loadModel({
- *   modelSrc: "/path/to/groot-multi.gguf",
- *   modelType: "vla",
- *   modelConfig: { embodiment: "libero_sim" }, // initial selection at load
- * });
- * const { hparams } = await vlaSetEmbodiment({ modelId, embodiment: "real_droid" });
- * console.log(hparams.selectedEmbodimentTag, hparams.numCameras);
- * ```
- */
-export declare function vlaSetEmbodiment(params: {
-    modelId: string;
-    embodiment: VlaEmbodimentSelection;
-}): Promise<{
-    hparams: VlaHparams;
-}>;
-//# sourceMappingURL=vla.d.ts.map