@qvac/sdk 0.15.0 → 0.17.0

package/dist/schemas/sdcpp-config.js

@@ -46,17 +46,35 @@ export const sdcppConfigSchema = z.object({
         .default('diffusion')
         .describe('Operation mode for the diffusion plugin. ' +
         "`'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from " +
-        'the primary model plus optional auxiliary text encoders, VAE, and ESRGAN ' +
+        'the primary model plus optional auxiliary text encoders, VAE, unconditional ' +
+        'diffusion model, and ESRGAN ' +
         'upscaler, and exposes diffusion({ ... }). ' +
         "`'upscale'` builds a standalone ESRGAN upscaler from the primary model " +
         'file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). ' +
-        "`'video'` builds a Wan `VideoStableDiffusion` pipeline and exposes video({ ... }). " +
+        "`'video'` builds a `VideoStableDiffusion` pipeline and exposes video({ ... }). " +
+        'The video layout is selected from the auxiliary sources: supplying ' +
+        '`embeddingsConnectorsModelSrc` loads the LTX-2 layout (Gemma text encoder ' +
+        'via `llmModelSrc` + video VAE + connectors, optional `audioVaeModelSrc` for ' +
+        'synchronized audio); otherwise the Wan layout is used (UMT5 text encoder ' +
+        'via `t5XxlModelSrc` + VAE). ' +
         'On React Native, loading the video model on-device will likely fail ' +
         'because the video diffusion models currently ' +
         'shipped by the SDK are too large to load on typical mobile devices; ' +
         'pass a `delegate` to `loadModel(...)` to run generation on a desktop peer instead.'),
     threads: z.number().optional(),
     device: z.enum(['gpu', 'cpu']).optional(),
+    'main-gpu': z
+        .union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
+        .optional()
+        .describe('GPU to pin when `device` is "gpu": a GPU-device index, "integrated", or ' +
+        '"dedicated" (the discrete GPU with the most VRAM). Omit to let the ' +
+        'backend choose the first enumerated device. Resolved against the ' +
+        "addon's own ggml device enumeration, so it cannot desync from the " +
+        'device list the backend actually uses. If an explicit request cannot ' +
+        'be satisfied (e.g. "integrated" with no integrated GPU, "dedicated" ' +
+        'with no discrete GPU, or an out-of-range index) the addon falls back ' +
+        'to CPU rather than substituting a different GPU. Stripped on mobile ' +
+        '(single-GPU devices).'),
     prediction: z
         .enum(['auto', 'eps', 'v', 'edm_v', 'flow', 'flux2_flow'])
         .optional()
@@ -116,17 +134,40 @@ export const sdcppConfigSchema = z.object({
         .describe('T5-XXL text encoder model — required for SD3'),
     llmModelSrc: modelSrcInputSchema
         .optional()
-        .describe('LLM text encoder model (e.g. Qwen3) — required for FLUX.2 [klein]'),
+        .describe('LLM text encoder model — required for FLUX.2 [klein] (Qwen3), ' +
+        'Ideogram 4 (Qwen3-VL), and LTX-2 video (Gemma).'),
     vaeModelSrc: modelSrcInputSchema
         .optional()
-        .describe('VAE decoder model — required for FLUX.2 [klein], optional for SDXL'),
+        .describe('VAE decoder model — required for FLUX.2 [klein], Ideogram 4, and ' +
+        'LTX-2 video (video VAE); optional for SDXL.'),
     highNoiseDiffusionModelSrc: modelSrcInputSchema
         .optional()
-        .describe('High-noise diffusion expert — required for Wan 2.2 mixture-of-experts video models'),
+        .describe('High-noise diffusion expert — required for Wan 2.2 A14B ' +
+        'mixture-of-experts video models, and the only thing that enables the ' +
+        'high_noise_* / moe_boundary request fields. Omit for single-expert ' +
+        'models such as Wan 2.1 and Wan 2.2 TI2V-5B.'),
+    uncondModelSrc: modelSrcInputSchema
+        .optional()
+        .describe('Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, ' +
+        'llmModelSrc (Qwen3-VL), vaeModelSrc, and a JSON-serialized structured ' +
+        'caption with explicit bounding boxes as the generation prompt. Plain-text ' +
+        "prompts produce degenerate output or the model's placeholder response."),
     clipVisionModelSrc: modelSrcInputSchema
         .optional()
         .describe('OpenCLIP ViT-H/14 weights (`clip_vision_h.safetensors`). Required for ' +
-        'Wan image-to-video (`img2vid`); omit for text-to-video-only pipelines.'),
+        'Wan image-to-video (`img2vid`); omit for text-to-video-only pipelines. ' +
+        'Not used by LTX-2 (its img2vid path needs no CLIP-vision projection).'),
+    audioVaeModelSrc: modelSrcInputSchema
+        .optional()
+        .describe('Audio VAE decoder model — LTX-2 video only. Enables the synchronized ' +
+        '48 kHz audio track muxed into the output AVI; omit for silent video. ' +
+        'Ignored by the Wan layout.'),
+    embeddingsConnectorsModelSrc: modelSrcInputSchema
+        .optional()
+        .describe('Text-embedding connector weights — required for LTX-2 video. Its ' +
+        'presence selects the LTX-2 video layout (Gemma text encoder via ' +
+        '`llmModelSrc` + video VAE via `vaeModelSrc` + these connectors) instead ' +
+        'of the Wan layout.'),
     upscaler: z
         .object({
         type: z
@@ -182,7 +223,27 @@ export const diffusionStatsSchema = z.object({
     generationMs: z
         .number()
         .optional()
-        .describe('Wall-clock time in milliseconds spent generating images.'),
+        .describe('Wall-clock time in milliseconds spent generating the output.'),
+    conditionerMs: z
+        .number()
+        .optional()
+        .describe('Time in milliseconds spent conditioning the prompt before denoising.'),
+    denoiseMs: z
+        .number()
+        .optional()
+        .describe('Time in milliseconds spent in the diffusion denoising loop.'),
+    vaeMs: z
+        .number()
+        .optional()
+        .describe('Time in milliseconds spent decoding diffusion latents with the VAE.'),
+    postProcessMs: z
+        .number()
+        .optional()
+        .describe('Time in milliseconds spent encoding, upscaling, muxing, and emitting outputs.'),
+    stepsPerSecond: z
+        .number()
+        .optional()
+        .describe('Diffusion denoising throughput in sampling steps per second.'),
     totalGenerationMs: z
         .number()
         .optional()
@@ -209,6 +270,11 @@ export const videoStatsSchema = diffusionStatsSchema
     .pick({
     modelLoadMs: true,
     generationMs: true,
+    conditionerMs: true,
+    denoiseMs: true,
+    vaeMs: true,
+    postProcessMs: true,
+    stepsPerSecond: true,
     totalGenerationMs: true,
     totalWallMs: true,
     totalSteps: true,
@@ -226,7 +292,16 @@ export const videoStatsSchema = diffusionStatsSchema
     fps: z
         .number()
         .optional()
-        .describe('Frames-per-second metadata for the most recent generated video.')
+        .describe('Frames-per-second metadata for the most recent generated video.'),
+    hasAudio: z
+        .boolean()
+        .optional()
+        .describe('True when the output AVI includes a muxed audio track (LTX-2 loaded ' +
+        'with audioVaeModelSrc), false otherwise.'),
+    audioSampleRate: z
+        .number()
+        .optional()
+        .describe('Sample rate (Hz) of the muxed audio track; 0 when there is no audio.')
 });
 export const diffusionStreamResponseSchema = z.object({
     type: z.literal('diffusionStream'),
@@ -395,14 +470,20 @@ const videoGenerationBaseSchema = z.object({
         .positive()
         .multipleOf(16)
         .optional()
-        .describe('Video width in pixels (must be a multiple of 16).'),
+        .describe('Video width in pixels (must be a multiple of 16). LTX-2 and Wan 2.2 ' +
+        'TI2V-5B additionally require a multiple of 32. LTX-2 is validated ' +
+        'against the loaded model before generation; the TI2V requirement is ' +
+        'enforced natively, derived from the loaded GGUF rather than its filename.'),
     height: z
         .number()
         .int()
         .positive()
         .multipleOf(16)
         .optional()
-        .describe('Video height in pixels (must be a multiple of 16).'),
+        .describe('Video height in pixels (must be a multiple of 16). LTX-2 and Wan 2.2 ' +
+        'TI2V-5B additionally require a multiple of 32. LTX-2 is validated ' +
+        'against the loaded model before generation; the TI2V requirement is ' +
+        'enforced natively, derived from the loaded GGUF rather than its filename.'),
     video_frames: z
         .number()
         .int()
@@ -410,7 +491,9 @@ const videoGenerationBaseSchema = z.object({
         message: 'video_frames must be an integer >= 5 of the form (4*k + 1)'
     })
         .optional()
-        .describe('Frame count for the generated video; must satisfy (4*k + 1), where k>=1.'),
+        .describe('Frame count for the generated video; must satisfy (4*k + 1), where k>=1. ' +
+        'LTX-2 additionally requires the stricter (8*k + 1) with a max of 257, ' +
+        'validated against the loaded model before generation.'),
     fps: z
         .number()
         .positive()
@@ -441,24 +524,34 @@ const videoGenerationBaseSchema = z.object({
         .int()
         .positive()
         .optional()
-        .describe('Wan 2.2 high-noise expert step count.'),
+        .describe('Wan 2.2 A14B high-noise expert step count. Requires a model loaded ' +
+        'with modelConfig.highNoiseDiffusionModelSrc; rejected otherwise. ' +
+        'Omit to let native routing derive the split from moe_boundary.'),
     high_noise_sampler: samplingMethodSchema
         .optional()
-        .describe('Wan 2.2 high-noise expert sampler.'),
+        .describe('Wan 2.2 A14B high-noise expert sampler. Requires a model loaded with ' +
+        'modelConfig.highNoiseDiffusionModelSrc; rejected otherwise.'),
     high_noise_scheduler: scheduleTypeSchema
         .optional()
-        .describe('Wan 2.2 high-noise expert scheduler.'),
-    high_noise_cfg_scale: z.number().optional().describe('Wan 2.2 high-noise expert CFG scale.'),
+        .describe('Wan 2.2 A14B high-noise expert scheduler. Requires a model loaded with ' +
+        'modelConfig.highNoiseDiffusionModelSrc; rejected otherwise.'),
+    high_noise_cfg_scale: z
+        .number()
+        .optional()
+        .describe('Wan 2.2 A14B high-noise expert CFG scale. Requires a model loaded with ' +
+        'modelConfig.highNoiseDiffusionModelSrc; rejected otherwise.'),
     high_noise_flow_shift: z
         .number()
         .optional()
-        .describe('Wan 2.2 high-noise expert flow shift override.'),
+        .describe('Wan 2.2 A14B high-noise expert flow shift override. Requires a model ' +
+        'loaded with modelConfig.highNoiseDiffusionModelSrc; rejected otherwise.'),
     moe_boundary: z
         .number()
         .min(0)
         .max(1)
         .optional()
-        .describe('Wan 2.2 mixture-of-experts boundary in [0, 1].'),
+        .describe('Wan 2.2 A14B mixture-of-experts boundary in [0, 1]. Requires a model ' +
+        'loaded with modelConfig.highNoiseDiffusionModelSrc; rejected otherwise.'),
     vace_strength: z.number().min(0).max(1).optional().describe('Control-frame guidance strength.'),
     control_frames: z
         .array(base64StringSchema)
@@ -474,6 +567,11 @@ const videoGenerationBaseSchema = z.object({
         .optional()
         .describe('VAE tile size override.'),
     vae_tile_overlap: z.number().optional().describe('VAE tile overlap override.'),
+    temporal_tiling: z
+        .boolean()
+        .optional()
+        .describe('LTX-2 only: tile the video VAE decode along the time axis to cap peak ' +
+        'VRAM for HD / long clips. No effect on Wan (spatial-only VAE).'),
     cache_mode: cacheModeSchema.optional().describe('Step-caching algorithm.'),
     cache_preset: z
         .string()
@@ -527,6 +625,56 @@ function refineVideoMode(data, ctx) {
     }
 }
 export const videoRequestSchema = videoRequestObjectSchema.superRefine(refineVideoMode);
+function refineLtxVideoRequest(data, ctx) {
+    if (data.width !== undefined && data.width % 32 !== 0) {
+        ctx.addIssue({
+            code: 'custom',
+            path: ['width'],
+            message: 'LTX-2 width must be a multiple of 32.'
+        });
+    }
+    if (data.height !== undefined && data.height % 32 !== 0) {
+        ctx.addIssue({
+            code: 'custom',
+            path: ['height'],
+            message: 'LTX-2 height must be a multiple of 32.'
+        });
+    }
+    if (data.video_frames !== undefined &&
+        (data.video_frames > 257 || (data.video_frames - 1) % 8 !== 0)) {
+        ctx.addIssue({
+            code: 'custom',
+            path: ['video_frames'],
+            message: 'LTX-2 video_frames must be at most 257 and satisfy (8*k + 1), where k>=1.'
+        });
+    }
+}
+export const ltxVideoRequestSchema = videoRequestSchema.superRefine(refineLtxVideoRequest);
+const wan22MoeMask = {
+    high_noise_steps: true,
+    high_noise_sampler: true,
+    high_noise_scheduler: true,
+    high_noise_cfg_scale: true,
+    high_noise_flow_shift: true,
+    moe_boundary: true
+};
+const wan22MoeSchema = videoGenerationBaseSchema.pick(wan22MoeMask);
+const wan22MoeFields = Object.keys(wan22MoeMask);
+function refineSingleExpertVideoRequest(data, ctx) {
+    for (const field of wan22MoeFields) {
+        if (data[field] === undefined)
+            continue;
+        ctx.addIssue({
+            code: 'custom',
+            path: [field],
+            message: `${field} is a Wan 2.2 A14B mixture-of-experts parameter and the loaded ` +
+                'model has no high-noise expert. Only a model loaded with ' +
+                'modelConfig.highNoiseDiffusionModelSrc routes to a second expert; ' +
+                'single-expert layouts (Wan 2.1, Wan 2.2 TI2V-5B, LTX-2) do not.'
+        });
+    }
+}
+export const singleExpertVideoRequestSchema = wan22MoeSchema.superRefine(refineSingleExpertVideoRequest);
 export const videoStreamRequestSchema = videoRequestObjectSchema
     .extend({ type: z.literal('videoStream') })
     .superRefine(refineVideoMode);