ima2-gen 3.7.2 → 3.8.0

package/bin/commands/video.js

@@ -78,6 +78,7 @@ const SPEC = {
         start: { type: "string" },
         end: { type: "string" },
         ref: { type: "string", repeatable: true },
+        voice: { type: "string", repeatable: true },
         "video-ref": { type: "string" },
         character: { type: "string" },
         out: { short: "o", type: "string" },
@@ -121,6 +122,9 @@ const HELP = `
         --start <generated-filename>    MCP start frame
         --end <generated-filename>      MCP end frame; requires --start
         --ref <file|@last|file:tag>     Grok image input or MCP tagged image reference
+        --voice <voice-id>              Preset voice for the subject (grok-imagine-video-1.5,
+                                        repeatable, max 3). e.g. eve, leo, rex, luna, atlas.
+                                        An unknown id returns the full list of valid voices
                                         Repeatable: Grok max 7, MCP max 3
         --video-ref <generated-filename> MCP V2V/restyle reference video
         --character <element-id|name>      MCP lanes only: character binding element
@@ -136,9 +140,9 @@ const HELP = `
         --duration <2..10>              Extension duration only. Default: 6
 
   Grok modes (auto-detected from --ref count):
-    0 refs  → text-to-video
-    1 ref   → image-to-video
-    2-7 refs → reference-to-video (max 10s duration)
+    0 refs   → text-to-video
+    1-7 refs → reference-to-video (max 720p). References guide the subject without
+               locking the first frame; use a node/continuity chain for a first frame.
 
   Examples:
     ima2 video "a cat playing piano"
@@ -214,8 +218,12 @@ async function videoEditCmd(argv) {
     const spec = { flags: { video: { type: "string" }, out: { short: "o", type: "string" }, output: { type: "string" }, json: { type: "boolean" }, timeout: { type: "string", default: String(VIDEO_CLIENT_TIMEOUT_SEC) }, server: { type: "string" }, help: { short: "h", type: "boolean" } } };
     const args = parseArgs(argv, spec);
     rejectUnknownFlags(args);
+    // Limits: 8.7s input is the owner's own measurement (docs/grok-video-i2v-research.md,
+    // 2026-05-30) and appears in no xAI doc — keep it. The 1.5 rejection and the
+    // ignored duration/resolution were live-probed against api.x.ai on 2026-08-20.
+    // Details + full probe table: skills/ima2/SKILL.md "Provenance of the limits above".
     if (args.help) {
-        out(`  ima2 video edit <prompt> --video <url|file_id|generated-file>\n\n  Edit existing video with text prompt (real V2V).\n  Model: grok-imagine-video only. Input: mp4, max 8.7s.\n\n  Options:\n        --video <value>   Source video HTTPS URL, xAI file_id, data URL, or generated filename (required)\n    -o, --out <file>      Download edited video to file\n        --output <file>   Alias for --out\n        --json            Print JSON result\n        --timeout <sec>   Default: 5400\n        --server <url>    Override server URL`);
+        out(`  ima2 video edit <prompt> --video <url|file_id|generated-file>\n\n  Edit existing video with text prompt (real V2V).\n  Model: grok-imagine-video only (1.5 rejects edits). Input: mp4, max 8.7s.\n  Output inherits the source video's duration and resolution (max 720p).\n\n  Options:\n        --video <value>   Source video HTTPS URL, xAI file_id, data URL, or generated filename (required)\n    -o, --out <file>      Download edited video to file\n        --output <file>   Alias for --out\n        --json            Print JSON result\n        --timeout <sec>   Default: 5400\n        --server <url>    Override server URL`);
         return;
     }
     const prompt = args.positional.join(" ");

package/bin/lib/videoMcp.js

@@ -1,7 +1,7 @@
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { config } from "../../config.js";
-import { deriveVideoMode, GROK_VIDEO_MODEL_15, GROK_VIDEO_MODEL_15_PREVIEW_ALIAS, validateVideoResolutionForRequest, } from "../../lib/imageModels.js";
+import { GROK_VIDEO_MODEL_15, GROK_VIDEO_MODEL_15_PREVIEW_ALIAS, validateVideoResolutionForRequest, } from "../../lib/imageModels.js";
 import { VIDEO_CLIENT_TIMEOUT_MS } from "../../lib/videoClientTimeouts.js";
 import { wasFlagPassed } from "./argsExplicit.js";
 import { resolveHistoryReference, resolveServer, request } from "./client.js";
@@ -94,9 +94,10 @@ function validateCoreOptions(args, refs, model) {
         die(2, "--aspect-ratio must be one of: 1:1, 16:9, 9:16, 4:3, 3:4, 3:2, 2:3, auto");
     if (refs.length > 7)
         die(2, "max 7 --ref attachments for video");
-    if (refs.length >= 2 && duration > 10)
-        die(2, "--duration must be between 1 and 10 when using 2 or more --ref attachments");
-    const check = validateVideoResolutionForRequest(model, resolution, deriveVideoMode(refs.length), { allowTextCanvasShim: true });
+    // `--ref` always fills the reference slot now, so any attachment means
+    // reference-to-video — the same conclusion the server reaches from the slot.
+    const mode = refs.length > 0 ? "reference-to-video" : "text-to-video";
+    const check = validateVideoResolutionForRequest(model, resolution, mode, { allowTextCanvasShim: true });
     if (!("ok" in check))
         die(2, check.error);
     return { duration, resolution, aspectRatio };
@@ -129,9 +130,15 @@ function coreBody(args, context, options, references, requestId) {
         body.sessionId = args.session;
     if (args.topic)
         body.topic = args.topic;
-    if (references.length === 1)
-        body.sourceImage = references[0];
-    else if (references.length > 1)
+    // Voice ids pass through unvalidated on purpose: xAI owns the roster (and custom
+    // voices), and its rejection names every accepted value.
+    const voices = (Array.isArray(args.voice) ? args.voice : args.voice ? [args.voice] : []);
+    if (voices.length > 0)
+        body.referenceAudios = voices.map((v) => String(v));
+    // `--ref` is the reference slot at any count; one reference guides the video without
+    // locking the first frame. Mapping a single --ref to sourceImage made the CLI disagree
+    // with the server contract and with the UI.
+    if (references.length > 0)
         body.referenceImages = references;
     return body;
 }

package/docs/migration/runtime-test-inventory.md

@@ -4,7 +4,7 @@ Generated by `npm run test:inventory` (script: `scripts/classify-tests.mjs`).
 
 _Tests considered "runtime-importing" if they import from `../lib/`, `../routes/`, `../bin/`, `../server`, or `../config`._
 
-Total: 366 (runtime: 163, contract: 203)
+Total: 371 (runtime: 168, contract: 203)
 
 ## Runtime-importing tests
 - `tests/agent-mode-auto-planner-contract.test.ts`
@@ -17,6 +17,7 @@ Total: 366 (runtime: 163, contract: 203)
 - `tests/agent-mode-ux-feedback-contract.test.ts`
 - `tests/agent-queue-store-contract.test.ts`
 - `tests/agent-video-intent.test.ts`
+- `tests/agent-video-reference-contract.test.ts`
 - `tests/agy-artifact-fallback.test.ts`
 - `tests/agy-cli.test.ts`
 - `tests/api-cache-policy.test.ts`
@@ -31,6 +32,7 @@ Total: 366 (runtime: 163, contract: 203)
 - `tests/backend-input-lan-hardening.test.ts`
 - `tests/background-presets.test.ts`
 - `tests/billing-source.test.ts`
+- `tests/capabilities-video-modes-contract.test.ts`
 - `tests/card-news-contract.test.ts`
 - `tests/card-news-template.test.ts`
 - `tests/classic-generate-async.test.ts`
@@ -163,7 +165,10 @@ Total: 366 (runtime: 163, contract: 203)
 - `tests/video-concat.test.ts`
 - `tests/video-history-item.test.ts`
 - `tests/video-motion-presets.test.ts`
+- `tests/video-ref2v-duration-contract.test.ts`
+- `tests/video-reference-audio-contract.test.ts`
 - `tests/video-request-contract.test.ts`
+- `tests/video-single-reference-mode-contract.test.ts`
 - `tests/videoArtifactPersistence.test.ts`
 - `tests/videoContinuity.test.ts`
 - `tests/videoExtendI2v.test.ts`

package/lib/agentGenerationPlanner.js

@@ -235,6 +235,9 @@ export function cleanVideoParams(value) {
     if (typeof input.aspectRatio === "string" && /^(auto|16:9|9:16|4:3|3:4|3:2|2:3|1:1)$/.test(input.aspectRatio)) {
         params.aspectRatio = input.aspectRatio;
     }
+    if (input.mode === "text-to-video" || input.mode === "image-to-video" || input.mode === "reference-to-video") {
+        params.mode = input.mode;
+    }
     return Object.keys(params).length > 0 ? params : null;
 }
 function cleanCommandName(value) {

package/lib/agentImageVideoGen.js

@@ -16,7 +16,7 @@ import { generateViaAtlasCloud } from "./atlasCloudImageAdapter.js";
 import { generateViaMinimax } from "./minimaxImageAdapter.js";
 import { DEFAULT_GROK_PLANNER_MODEL } from "../config.js";
 import { generateVideoViaGrok } from "./grokVideoAdapter.js";
-import { GROK_VIDEO_MODEL_15, GROK_VIDEO_MODEL_BASE, resolveGrokQualityModel } from "./imageModels.js";
+import { GROK_VIDEO_MODEL_15, GROK_VIDEO_MODEL_BASE, resolveGrokQualityModel, validateVideoResolutionForRequest } from "./imageModels.js";
 import { parseVideoParams } from "./agentGenerationPlanner.js";
 import { isVideoGenerationError, normalizeVideoGenerationRequest, } from "./videoGenerationRequest.js";
 import { appendAgentTurn, getAgentImages, getAgentSession, importAgentImage, } from "./agentStore.js";
@@ -198,17 +198,29 @@ export async function runAgentVideoGeneration(ctx, sessionId, prompt, options =
     }
     const requestId = options.requestId ?? `agent_video_${ulid()}`;
     const startedAt = Date.now();
-    // Auto I2V: if session has a last image, use it as source
+    // Attach the session's current image unless the planner said not to. The image path
+    // has always honored this policy; the video path ignored it, so "make something new"
+    // still got the previous image welded on as the first frame.
+    //
+    // The planner also chooses what the image MEANS. A first frame reproduces the source
+    // shot; a reference carries the subject into a new one. Chat has no reference tray,
+    // so without the planner saying which, reference-to-video was unreachable here.
+    // devlog/_plan/260820_grok15_multi_reference_video/020_agent_reference_loss.md
+    const requestedMode = options.videoParams?.mode;
+    const attachPolicy = options.sourceImagePolicy ?? "auto";
     let sourceImage;
-    let mode = "text-to-video";
-    if (session.lastImageId) {
+    let referenceImages;
+    if (attachPolicy !== "none" && session.lastImageId) {
         const images = getAgentImages(sessionId);
         const lastImage = images.find((img) => img.id === session.lastImageId);
         if (lastImage?.filename && !lastImage.filename.endsWith(".mp4")) {
             try {
                 const { loadAssetB64 } = await import("./nodeStore.js");
-                sourceImage = await loadAssetB64(ctx.rootDir, lastImage.filename, ctx.config.storage.generatedDir);
-                mode = "image-to-video";
+                const b64 = await loadAssetB64(ctx.rootDir, lastImage.filename, ctx.config.storage.generatedDir);
+                if (requestedMode === "reference-to-video")
+                    referenceImages = [b64];
+                else
+                    sourceImage = b64;
             }
             catch { /* fallback to T2V */ }
         }
@@ -219,8 +231,9 @@ export async function runAgentVideoGeneration(ctx, sessionId, prompt, options =
     // validation cannot drift between "generate from chat" and "generate from the app".
     const normalized = normalizeVideoGenerationRequest({
         prompt,
-        mode,
         sourceImage,
+        ...(referenceImages ? { referenceImages } : {}),
+        ...(requestedMode ? { mode: requestedMode } : {}),
         duration: options.videoParams?.duration ?? parsedParams.duration,
         resolution: options.videoParams?.resolution ?? parsedParams.resolution,
         aspectRatio: options.videoParams?.aspectRatio ?? parsedParams.aspectRatio,
@@ -236,11 +249,26 @@ export async function runAgentVideoGeneration(ctx, sessionId, prompt, options =
     const videoModel = videoParams.resolution === "1080p"
         ? GROK_VIDEO_MODEL_15
         : GROK_VIDEO_MODEL_BASE;
+    // Reference-to-video has no 1080p (xAI returns 400). Say so instead of quietly
+    // downgrading the resolution — silently altering the request is the defect this
+    // work-phase exists to remove. Same helper the HTTP route uses, so the two surfaces
+    // cannot drift apart again.
+    const resolutionCheck = validateVideoResolutionForRequest(videoModel, videoParams.resolution, videoParams.mode, {
+        allowTextCanvasShim: true,
+    });
+    if (!("ok" in resolutionCheck)) {
+        throw Object.assign(new Error(resolutionCheck.error), {
+            status: resolutionCheck.status,
+            code: resolutionCheck.code,
+        });
+    }
     options.onProgressStage?.("requesting");
     const result = await generateVideoViaGrok(prompt, ctx, {
         model: videoModel,
-        mode: videoParams.mode === "reference-to-video" ? "text-to-video" : videoParams.mode,
+        mode: videoParams.mode,
         sourceImage,
+        ...(videoParams.referenceImages ? { referenceImages: videoParams.referenceImages } : {}),
+        ...(videoParams.referenceAudios ? { referenceAudios: videoParams.referenceAudios } : {}),
         duration: videoParams.duration,
         resolution: videoParams.resolution,
         aspectRatio: videoParams.aspectRatio,

package/lib/agentPlannerModel.js

@@ -27,7 +27,7 @@ function buildPlannerDeveloperPrompt(hasSourceImage, imageCount) {
         `- Last image available as image-to-video source: ${hasSourceImage ? "yes" : "no"}`,
         "",
         "Decide ONE plan and respond with ONLY a JSON object (no prose, no code fences):",
-        '{"mode":"single|fanout|video|question|errors","prompts":["..."],"plannedVariants":1,"plannedParallelism":1,"sourceImagePolicy":"none|current|auto","videoParams":{"duration":5,"resolution":"480p","aspectRatio":"auto"},"assistantText":"...","reason":"short reason"}',
+        '{"mode":"single|fanout|video|question|errors","prompts":["..."],"plannedVariants":1,"plannedParallelism":1,"sourceImagePolicy":"none|current|auto","videoParams":{"duration":5,"resolution":"480p","aspectRatio":"auto","mode":"text-to-video|image-to-video|reference-to-video"},"assistantText":"...","reason":"short reason"}',
         "",
         "Rules:",
         "- You are a conversational assistant first. Generate media ONLY when the user clearly asks you to create or edit an image/video. Everything else (questions, chat, greetings, feedback, follow-ups) is mode question.",
@@ -37,6 +37,7 @@ function buildPlannerDeveloperPrompt(hasSourceImage, imageCount) {
         "- sourceImagePolicy for single/fanout: use current only when the user explicitly asks to use/edit/modify/transform/reference the current image, including '이 이미지', '현재 이미지', '방금 그거', '참조', 'reference', 'i2i', 'image-to-image', '유지해서'.",
         "- sourceImagePolicy for plain image requests with no explicit reference wording is none.",
         "- mode video: one video via ima2.generate_video. Choose it only when the user asks to CREATE a video. prompts has exactly 1 entry. Extract duration (1-15 s), resolution (480p|720p|1080p), aspectRatio (auto|1:1|16:9|9:16|4:3|3:4|3:2|2:3) from the request into videoParams; omit fields the user did not specify. 1080p uses Grok Video 1.5; prompt-only requests are valid because the server injects a white-canvas I2V shim.",
+        "- videoParams.mode decides what the session's current image is FOR. Use image-to-video when the user wants that exact picture animated or continued ('이 이미지를 움직여줘', 'animate this', 'make it move'): it becomes the first frame. Use reference-to-video when the user wants the subject, character, outfit or place carried into a DIFFERENT scene ('이 캐릭터로 다른 장면', 'same person at the beach', 'use this outfit'): it guides the video without locking the opening shot. reference-to-video cannot use 1080p. Omit mode when the user did not imply either.",
         "- mode question: the user is NOT requesting generation — a question (capabilities, how-to, status), small talk, a greeting, or feedback — e.g. '영상 생성가능하니?', 'can you make videos?', '고마워'. prompts must be []. Write the full answer in assistantText. Mentioning a media word like 'video' or '영상' inside a question does NOT make it a generation request.",
         "- mode errors: the user is asking why a previous generation failed or about recent errors. prompts must be [].",
         "- assistantText: REQUIRED for every mode, written in the user's language. For question/errors it is the full reply. For single/fanout/video it is a short natural chat reply telling the user what you are creating (1-2 sentences, no markdown headings).",

package/lib/agentRuntime.js

@@ -66,6 +66,9 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
         return runAgentVideoGeneration(ctx, sessionId, plan.prompts[0] ?? prompt, {
             ...options,
             videoParams: plan.videoParams ?? options.videoParams ?? null,
+            // The image path has always forwarded this (see below); the video path did not,
+            // so a planner that chose "none" was overruled by whatever image came last.
+            sourceImagePolicy: plan.sourceImagePolicy ?? "auto",
             assistantText: preludeSent ? null : plan.assistantText,
             requestId: options.requestId ?? `agent_video_${ulid()}`,
             skipUserTurn: true,

package/lib/capabilities.js

@@ -5,6 +5,7 @@ import { buildCatalog, catalogSummary } from "./contracts/catalog.js";
 import { loadAllBundledSnapshots } from "./mcp/snapshotStore.js";
 import { KEY_TO_ENV, WRITABLE_CONFIG_KEYS } from "./configKeys.js";
 import { DEFAULT_IMAGE_QUALITY, VALID_IMAGE_QUALITIES } from "./oauthNormalize.js";
+import { MAX_REF2V_REFERENCES, MAX_REFERENCE_AUDIOS, MAX_VIDEO_DURATION, MIN_VIDEO_DURATION } from "./imageModels.js";
 import { deriveProviderIds } from "./providers/derive.js";
 const VALID_MODES = ["auto", "direct"];
 const VALID_PROVIDERS = ["auto", ...deriveProviderIds()];
@@ -71,6 +72,45 @@ export function buildIma2Capabilities({ appConfig = runtimeConfigDefault, packag
                 aspectRatios: ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "auto"],
                 durationRange: [1, 15],
                 maxReferences: 7,
+                // Flat numbers above describe the widest case, which is not what any single
+                // request is allowed to do. A client that reads only those draws controls the
+                // server will reject. These per-mode entries come from the same constants the
+                // request path enforces, so the advertisement cannot drift from the behavior.
+                // Verified against api.x.ai on 2026-08-20:
+                // devlog/_plan/260820_grok15_multi_reference_video/000_research.md
+                modes: {
+                    "text-to-video": {
+                        maxReferences: 0,
+                        durationRange: [MIN_VIDEO_DURATION, MAX_VIDEO_DURATION],
+                        resolutions: ["480p", "720p", "1080p"],
+                        notes: "1080p on grok-imagine-video-1.5 goes through the white-canvas shim.",
+                    },
+                    "image-to-video": {
+                        maxReferences: 1,
+                        durationRange: [MIN_VIDEO_DURATION, MAX_VIDEO_DURATION],
+                        resolutions: ["480p", "720p", "1080p"],
+                        notes: "The source image becomes the first frame.",
+                    },
+                    "reference-to-video": {
+                        maxReferences: MAX_REF2V_REFERENCES,
+                        durationRange: [MIN_VIDEO_DURATION, MAX_VIDEO_DURATION],
+                        resolutions: ["480p", "720p"],
+                        notes: "References guide the subject without locking the first frame. 1080p is rejected upstream.",
+                    },
+                },
+                referenceAudio: {
+                    maxVoices: MAX_REFERENCE_AUDIOS,
+                    models: ["grok-imagine-video-1.5"],
+                    // Not the allowlist: xAI owns the roster and accepts custom voice ids too. An
+                    // unknown id comes back as a 400 naming every voice it will take.
+                    knownPresets: [
+                        "ara", "eve", "leo", "rex", "sal", "carina", "zagan", "helix", "orion",
+                        "luna", "iris", "altair", "zenith", "perseus", "helios", "lux", "kepler",
+                        "rigel", "cosmo", "celeste", "ursa", "sirius", "lumen", "castor", "naksh",
+                        "atlas",
+                    ],
+                    presetsAreAuthoritative: false,
+                },
             },
             reasoningEfforts: toArray(appConfig.imageModels.validReasoningEfforts),
             quality: toArray(VALID_IMAGE_QUALITIES),

package/lib/errors/providerMap.js

@@ -45,6 +45,8 @@ export const PROVIDER_ERROR_MAP = {
     GROK_VIDEO_EMPTY_RESPONSE: "INTERNAL_STATE_ERROR",
     GROK_VIDEO_EXPIRED: "MODEL_UNAVAILABLE",
     GROK_VIDEO_FAILED: "INTERNAL_STATE_ERROR",
+    GROK_VIDEO_AUDIO_TOO_MANY: "CAPABILITY_UNSUPPORTED",
+    GROK_VIDEO_AUDIO_UNSUPPORTED_MODEL: "CAPABILITY_UNSUPPORTED",
     GROK_VIDEO_INVALID_MODE: "CAPABILITY_UNSUPPORTED",
     GROK_VIDEO_MODERATION_BLOCKED: "CONTENT_REJECTED",
     GROK_VIDEO_POLL_FAILED: "NETWORK_FAILURE",

package/lib/grokVideoAdapter.js

@@ -5,7 +5,7 @@ import { detectImageMimeFromB64 } from "./refs.js";
 import { aspectToCanvas, generateWhiteCanvasB64 } from "./grokVideoCanvas.js";
 import { downloadVideo } from "./grokVideoDownload.js";
 import { buildGrokVideoPlannerSystemPrompt, composeFallbackVideoPrompt, formatDurationPacingGuidance } from "./grokVideoPlannerPrompt.js";
-import { GROK_VIDEO_MODEL_15, GROK_VIDEO_MODEL_15_PREVIEW_ALIAS, GROK_VIDEO_MODEL_BASE, MAX_REF2V_REFERENCES, validateVideoResolutionForRequest, } from "./imageModels.js";
+import { GROK_VIDEO_MODEL_15, GROK_VIDEO_MODEL_15_PREVIEW_ALIAS, GROK_VIDEO_MODEL_BASE, MAX_REF2V_REFERENCES, MAX_REFERENCE_AUDIOS, validateVideoResolutionForRequest, } from "./imageModels.js";
 import { formatVideoContinuityForPlanner } from "./videoContinuity.js";
 import { DEFAULT_GROK_PLANNER_MODEL } from "../config.js";
 import { videoConfig, videoEndpoint, withTimeoutSignal, } from "./grokVideoShared.js";
@@ -58,6 +58,12 @@ export function buildGrokVideoPlannerPayload(prompt, opts) {
         : isI2V
             ? "This is image-to-video: preserve subject identity and composition unless asked otherwise, and use the source image as the first frame / starting point."
             : "This is text-to-video: describe motion, camera, and action clearly.";
+    // Voices only bind to a speaker if the prompt says where they go, and the planner
+    // rewrites the prompt — so it has to know the tag convention.
+    const voiceCount = opts.referenceAudios?.length ?? 0;
+    const voiceGuidance = voiceCount > 0
+        ? `${voiceCount} preset voice${voiceCount > 1 ? "s are" : " is"} attached, referred to as <AUDIO_0>..<AUDIO_${voiceCount - 1}>. Say who speaks with which voice in the prompt (for example "the person from <IMAGE_1> speaks with <AUDIO_0>"); a voice nobody is assigned to will not be used.`
+        : "";
     const lineageText = formatVideoContinuityForPlanner(opts.continuityLineage);
     const userContent = [
         {
@@ -68,6 +74,7 @@ export function buildGrokVideoPlannerPayload(prompt, opts) {
                 continuity,
                 lineageText ? `Authoritative continuation context:\n${lineageText}` : "Authoritative continuation context: none.",
                 formatDurationPacingGuidance(opts.duration, opts.mode, opts.resolution),
+                ...(voiceGuidance ? [voiceGuidance] : []),
                 opts.searchSummary ? `Mandatory web-search brief:\n${opts.searchSummary}` : "Mandatory web-search brief: unavailable.",
                 ...(opts.backgroundConstraint ? [opts.backgroundConstraint] : []),
                 "Return the generate_video.prompt argument in English only, except for exact visible text the user explicitly requested.",
@@ -200,6 +207,7 @@ export async function planGrokVideo(prompt, ctx, options = {}) {
             searchSummary,
             sourceImageUrl: options.sourceImage ? sourceImageUrl(options.sourceImage, options.sourceMime) : undefined,
             referenceImageUrls,
+            ...(options.referenceAudios?.length ? { referenceAudios: options.referenceAudios } : {}),
             continuityLineage: options.continuityLineage,
             backgroundConstraint: options.backgroundConstraint,
         });
@@ -275,13 +283,28 @@ export async function planGrokVideo(prompt, ctx, options = {}) {
 }
 export function buildVideoGenerationPayload(plan, opts) {
     const model = canonicalVideoModel(opts.model);
+    const voices = opts.referenceAudios ?? [];
+    if (voices.length > 0) {
+        // Preset voices exist only on 1.5. Attaching them to the base model earns a 400 from
+        // xAI; dropping them earns a video without the voice the user asked for. Say no.
+        if (model !== GROK_VIDEO_MODEL_15) {
+            throw grokError(`reference audio requires ${GROK_VIDEO_MODEL_15}`, 400, "GROK_VIDEO_AUDIO_UNSUPPORTED_MODEL");
+        }
+        if (voices.length > MAX_REFERENCE_AUDIOS) {
+            throw grokError(`at most ${MAX_REFERENCE_AUDIOS} reference voices`, 400, "GROK_VIDEO_AUDIO_TOO_MANY");
+        }
+    }
     if (plan.mode === "image-to-video" && !opts.sourceImageUrl) {
         throw grokError("image-to-video requires a source image", 400, "GROK_VIDEO_INVALID_MODE");
     }
     const refs = opts.referenceImageUrls ?? [];
     if (plan.mode === "reference-to-video") {
-        if (refs.length < 2)
-            throw grokError("reference-to-video requires at least 2 reference images", 400, "GROK_VIDEO_INVALID_MODE");
+        // One reference is a legitimate reference-to-video request: it guides the subject
+        // without locking the first frame, which is what distinguishes it from
+        // image-to-video. Verified against api.x.ai on 2026-08-20 (000_research.md).
+        // Zero references is still nonsense — that is text-to-video wearing the wrong name.
+        if (refs.length < 1)
+            throw grokError("reference-to-video requires at least 1 reference image", 400, "GROK_VIDEO_INVALID_MODE");
         if (refs.length > MAX_REF2V_REFERENCES)
             throw grokError(`reference-to-video allows at most ${MAX_REF2V_REFERENCES} reference images`, 400, "GROK_VIDEO_REF_TOO_MANY");
         if (opts.sourceImageUrl)
@@ -298,6 +321,8 @@ export function buildVideoGenerationPayload(plan, opts) {
         payload.image = { url: opts.sourceImageUrl };
     if (plan.mode === "reference-to-video")
         payload.reference_images = refs.map((url) => ({ url }));
+    if (voices.length > 0)
+        payload.reference_audios = voices.map((voiceId) => ({ voice_id: voiceId }));
     return payload;
 }
 export async function startVideoRequest(ctx, payload, options) {
@@ -337,6 +362,7 @@ export async function generateVideoViaGrok(prompt, ctx, options = {}) {
     const model = canonicalVideoModel(options.model || cfg.model);
     const srcUrl = options.sourceImage ? sourceImageUrl(options.sourceImage, options.sourceMime) : undefined;
     const refUrls = (options.referenceImages ?? []).map((img) => sourceImageUrl(img, undefined));
+    const voices = options.referenceAudios ?? [];
     options.onEvent?.({ phase: "planning" });
     const plan = options.plannedPrompt
         ? {
@@ -356,18 +382,21 @@ export async function generateVideoViaGrok(prompt, ctx, options = {}) {
         const { width, height } = aspectToCanvas(plan.aspectRatio, plan.resolution);
         const whiteCanvas = await generateWhiteCanvasB64(width, height);
         const canvasSrcUrl = `data:image/png;base64,${whiteCanvas}`;
-        effectivePayload = buildVideoGenerationPayload({ ...plan, mode: "image-to-video", prompt: `[Technical note: the attached image is a blank white canvas used as a technical placeholder for text-to-video generation. It is NOT a meaningful source frame. Ignore it completely and generate a fresh scene from scratch.]\n\n${plan.prompt}` }, { model, sourceImageUrl: canvasSrcUrl, referenceImageUrls: [] });
+        effectivePayload = buildVideoGenerationPayload({ ...plan, mode: "image-to-video", prompt: `[Technical note: the attached image is a blank white canvas used as a technical placeholder for text-to-video generation. It is NOT a meaningful source frame. Ignore it completely and generate a fresh scene from scratch.]\n\n${plan.prompt}` }, { model, sourceImageUrl: canvasSrcUrl, referenceImageUrls: [], ...(voices.length ? { referenceAudios: voices } : {}) });
         logEvent("grok", "video:1.5-t2v-canvas", { requestId: options.requestId, width, height });
     }
     else {
-        effectivePayload = buildVideoGenerationPayload(plan, { model, sourceImageUrl: srcUrl, referenceImageUrls: refUrls });
+        effectivePayload = buildVideoGenerationPayload(plan, { model, sourceImageUrl: srcUrl, referenceImageUrls: refUrls, ...(voices.length ? { referenceAudios: voices } : {}) });
     }
     try {
         xaiVideoRequestId = await startVideoRequest(ctx, effectivePayload, options);
     }
     catch (e) {
-        // Fallback: if 1.5-preview still fails, retry with base model
-        if (model !== GROK_VIDEO_MODEL_BASE && e?.status === 400) {
+        // Fallback: if 1.5-preview still fails, retry with base model.
+        // Not when voices are attached: the base model rejects reference_audios outright, so
+        // the retry would only replace one 400 with a more confusing one. Dropping the voice
+        // to make the call succeed would hand back a video missing what was asked for.
+        if (model !== GROK_VIDEO_MODEL_BASE && e?.status === 400 && voices.length === 0) {
             effectiveModel = GROK_VIDEO_MODEL_BASE;
             const fallbackPayload = buildVideoGenerationPayload(plan, { model: effectiveModel, sourceImageUrl: srcUrl, referenceImageUrls: refUrls });
             xaiVideoRequestId = await startVideoRequest(ctx, fallbackPayload, options);

package/lib/imageModels.js

@@ -141,10 +141,23 @@ export const VALID_VIDEO_ASPECT_RATIOS = new Set([
 ]);
 export const MIN_VIDEO_DURATION = 1;
 export const MAX_VIDEO_DURATION = 15;
-// reference-to-video (xAI): up to 7 reference images, max 10s duration.
+// reference-to-video (xAI): up to 7 reference images (8 -> 400), 1-15s, 720p max.
+// Verified against api.x.ai on 2026-08-20:
+// devlog/_plan/260820_grok15_multi_reference_video/000_research.md
 export const MAX_REF2V_REFERENCES = 7;
-export const MAX_REF2V_DURATION = 10;
-// Mode is derived purely from the number of attached reference images.
+// reference_audios: preset voices, grok-imagine-video-1.5 only. 4 -> 400.
+export const MAX_REFERENCE_AUDIOS = 3;
+/**
+ * Count-only fallback for callers that have lost track of WHICH slot each image
+ * arrived in.
+ *
+ * A single image is ambiguous on its own: as a first frame it means image-to-video,
+ * as a reference it means reference-to-video, and the count cannot tell them apart.
+ * This returns the historical default (image-to-video) so old callers keep their
+ * behavior — it is a fallback, not a ceiling. Callers that DO know the slot should
+ * use `deriveVideoMode` in lib/videoGenerationRequest.ts, which reads intent from
+ * the field the caller chose.
+ */
 export function deriveVideoMode(refCount) {
     if (refCount >= 2)
         return "reference-to-video";
@@ -152,10 +165,6 @@ export function deriveVideoMode(refCount) {
         return "image-to-video";
     return "text-to-video";
 }
-// Clamp duration to the reference-to-video ceiling; other modes keep their value.
-export function clampVideoDuration(duration, mode) {
-    return mode === "reference-to-video" ? Math.min(duration, MAX_REF2V_DURATION) : duration;
-}
 export function isGrokVideoModel(value) {
     return typeof value === "string" && VALID_GROK_VIDEO_MODELS.has(value);
 }

package/lib/videoGenerationRequest.js

@@ -69,6 +69,7 @@ export function normalizeVideoGenerationRequest(input) {
             ...(input.sourceAssetId ? { sourceAssetId: input.sourceAssetId } : {}),
             ...(input.referenceImages?.length ? { referenceImages: input.referenceImages } : {}),
             ...(input.referenceFilenames?.length ? { referenceFilenames: input.referenceFilenames } : {}),
+            ...(input.referenceAudios?.length ? { referenceAudios: input.referenceAudios } : {}),
             ...(input.continueFromVideo ? { continueFromVideo: input.continueFromVideo } : {}),
             ...(input.topic ? { topic: input.topic } : {}),
             ...(input.storyboard ? { storyboard: true } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ima2-gen",
-  "version": "3.7.2",
+  "version": "3.8.0",
   "packageManager": "npm@11.18.0",
   "description": "Local-first visual generation runtime and studio for people and coding agents, with reproducible image and video workflows across multiple providers.",
   "type": "module",
@@ -119,5 +119,5 @@
     "tsx": "^4.23.12",
     "typescript": "^5.9.3"
   },
-  "gitHead": "8f72a5b19e9aa58c7ec9ea540e988b3edc8ae4f5"
+  "gitHead": "780f6da658d0ae69b5c5a22d946c4742c32a4b9b"
 }

package/routes/video.js

@@ -15,7 +15,7 @@ import { getVideoSeriesChain } from "../lib/videoSeriesChain.js";
 import { ACTIVE_VIDEO_PROMPT_GUIDANCE, appendVideoContinuityEntry, lineageFromVideoMetadata, normalizeVideoContinuityLineage, readVideoSidecar, requireActiveVideoPrompt, safeGeneratedVideoFilename, } from "../lib/videoContinuity.js";
 import { extractGeneratedVideoFrameB64 } from "../lib/videoFrameExtract.js";
 import { errorEnvelopeFields } from "../lib/errors/envelope.js";
-import { normalizeGrokVideoModel, normalizeVideoResolution, normalizeVideoAspectRatio, normalizeVideoDuration, deriveVideoMode, clampVideoDuration, MAX_REF2V_REFERENCES, validateVideoResolutionForRequest, } from "../lib/imageModels.js";
+import { normalizeGrokVideoModel, normalizeVideoResolution, normalizeVideoAspectRatio, normalizeVideoDuration, deriveVideoMode, MAX_REF2V_REFERENCES, MAX_REFERENCE_AUDIOS, validateVideoResolutionForRequest, } from "../lib/imageModels.js";
 import { errInfo } from "../lib/errInfo.js";
 import { requireRuntimeContext } from "../lib/runtimeContext.js";
 import { generateVideoThumbnail } from "../lib/videoThumb.js";
@@ -287,7 +287,10 @@ export function registerVideoRoutes(app, ctxRaw) {
             refInputs.push(...elementResolvedRefs.map((image) => ({ image, source: "composer" })));
             let resolved;
             try {
-                const all = await Promise.all(refInputs.map((r) => resolveSourceImage(ctx, r.image, r.filename)));
+                const all = await Promise.all(refInputs.map(async (r) => ({
+                    ...(await resolveSourceImage(ctx, r.image, r.filename)),
+                    source: r.source,
+                })));
                 resolved = all.filter((r) => Boolean(r.b64));
             }
             catch (e) {
@@ -296,8 +299,28 @@ export function registerVideoRoutes(app, ctxRaw) {
             if (resolved.length > MAX_REF2V_REFERENCES)
                 return fail(400, "GROK_VIDEO_REF_TOO_MANY", `at most ${MAX_REF2V_REFERENCES} reference images`);
             const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http") ? req.body.providerUrl : null;
-            const mode = incomingProviderUrl ? "image-to-video" : deriveVideoMode(resolved.length);
-            const duration = clampVideoDuration(durationCheck.duration, mode);
+            // Which slot an image arrived in IS the caller's intent, and a bare count throws
+            // that away. A composer reference means "guide the video with this" even when
+            // there is only one of them; a node/continuity image means "start from this
+            // frame". Deriving from the count alone forced every single composer reference
+            // into image-to-video, so the one thing the tray is named for was unreachable.
+            // devlog/_plan/260820_grok15_multi_reference_video/030_single_ref_mode_choice.md
+            const composerRefCount = resolved.filter((r) => r.source === "composer").length;
+            const requestedMode = typeof req.body?.mode === "string" ? req.body.mode : null;
+            const derivedMode = composerRefCount > 0
+                ? "reference-to-video"
+                : deriveVideoMode(resolved.length);
+            const mode = incomingProviderUrl
+                ? "image-to-video"
+                : (requestedMode === "reference-to-video" || requestedMode === "image-to-video" || requestedMode === "text-to-video")
+                    ? requestedMode
+                    : derivedMode;
+            // An explicit reference-to-video with nothing to reference would ship an empty
+            // reference_images array and fail upstream with a less useful message.
+            if (mode === "reference-to-video" && resolved.length === 0) {
+                return fail(400, "GROK_VIDEO_INVALID_MODE", "reference-to-video requires at least 1 reference image");
+            }
+            const duration = durationCheck.duration;
             const resolutionModeCheck = validateVideoResolutionForRequest(modelCheck.model, resolutionCheck.resolution, mode, {
                 allowTextCanvasShim: true,
             });
@@ -356,6 +379,15 @@ export function registerVideoRoutes(app, ctxRaw) {
                 + (backgroundPreset ? ` ${backgroundPromptSuffix(backgroundPreset, "video")}` : "");
             const plannerModel = typeof req.body?.plannerModel === "string" ? req.body.plannerModel.trim() : undefined;
             const directApiKey = provider === "grok-api" ? ctx.xaiApiKey : undefined;
+            // Only the shape is checked here. Which voice ids exist is xAI's to answer, and its
+            // 400 names every valid voice — a list we would only get wrong, and which cannot
+            // include the caller's custom voices anyway.
+            const referenceAudios = toArray(req.body?.referenceAudios)
+                .map((voice) => (typeof voice === "string" ? voice.trim() : ""))
+                .filter((voice) => voice.length > 0);
+            if (referenceAudios.length > MAX_REFERENCE_AUDIOS) {
+                return fail(400, "GROK_VIDEO_AUDIO_TOO_MANY", `at most ${MAX_REFERENCE_AUDIOS} reference voices`);
+            }
             const result = await generateVideoViaGrok(effectivePrompt, ctx, {
                 model: modelCheck.model,
                 mode,
@@ -364,6 +396,7 @@ export function registerVideoRoutes(app, ctxRaw) {
                 aspectRatio: aspectCheck.aspectRatio,
                 sourceImage: sourceB64,
                 referenceImages,
+                ...(referenceAudios.length ? { referenceAudios } : {}),
                 signal: cancelController.signal,
                 requestId,
                 continuityLineage: parentLineage,