ima2-gen 2.0.1 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -0
- package/README.md +10 -1
- package/bin/commands/backfillThumbs.js +6 -0
- package/bin/commands/gen.js +6 -0
- package/bin/ima2.js +14 -10
- package/docs/API.md +131 -8
- package/docs/CLI.md +2 -1
- package/docs/FAQ.ko.md +16 -0
- package/docs/FAQ.md +30 -0
- package/docs/README.ko.md +7 -3
- package/docs/migration/runtime-test-inventory.md +15 -1
- package/lib/agentImageVideoGen.js +261 -0
- package/lib/agentRuntime.js +7 -262
- package/lib/agyImageAdapter.js +35 -8
- package/lib/errorClassify.js +8 -7
- package/lib/eventBus.js +71 -0
- package/lib/geminiApiImageAdapter.js +16 -20
- package/lib/generationErrors.js +3 -1
- package/lib/grokImageAdapter.js +68 -129
- package/lib/grokImageCore.js +153 -0
- package/lib/grokMultimodeAdapter.js +5 -3
- package/lib/grokVideoCanvas.js +13 -0
- package/lib/grokVideoPlannerPrompt.js +53 -6
- package/lib/historyList.js +1 -0
- package/lib/inflight.js +54 -17
- package/lib/multimodeHelpers.js +10 -0
- package/lib/nodeHelpers.js +59 -0
- package/lib/oauthProxy/prompts.js +30 -36
- package/lib/promptBuilder/systemPrompt.js +2 -5
- package/lib/promptSafetyPolicy.js +1 -5
- package/lib/responsesFallback.js +2 -1
- package/lib/routeHelpers.js +44 -0
- package/lib/ssePublish.js +12 -0
- package/lib/storyboardPrefix.js +28 -0
- package/lib/thumbBackfill.js +16 -5
- package/package.json +4 -1
- package/routes/agy.js +44 -0
- package/routes/auth.js +6 -2
- package/routes/edit.js +7 -1
- package/routes/events.js +78 -0
- package/routes/generate.js +99 -127
- package/routes/index.js +4 -0
- package/routes/multimode.js +99 -56
- package/routes/nodes.js +59 -103
- package/routes/video.js +100 -17
- package/skills/ima2/SKILL.md +98 -21
- package/ui/dist/.vite/manifest.json +12 -12
- package/ui/dist/assets/{AgentWorkspace-CYv84Rus.js → AgentWorkspace-Dth6YijN.js} +1 -1
- package/ui/dist/assets/{CardNewsWorkspace-Dqyc1WZ1.js → CardNewsWorkspace-Dav3K5CT.js} +1 -1
- package/ui/dist/assets/{NodeCanvas-ChEXzQbb.js → NodeCanvas-C4ifFzB1.js} +1 -1
- package/ui/dist/assets/{PromptBuilderPanel-B95ZufnR.js → PromptBuilderPanel-CEcyU9PL.js} +1 -1
- package/ui/dist/assets/{PromptImportDialog-DGOwFQET.js → PromptImportDialog-CgQ94Gth.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-CgvdnR49.js → PromptImportDiscoverySection-CuzyzbNI.js} +1 -1
- package/ui/dist/assets/{PromptImportFolderSection-CfUye9J8.js → PromptImportFolderSection-DHLGlO6l.js} +1 -1
- package/ui/dist/assets/{PromptLibraryPanel-B9kndPw1.js → PromptLibraryPanel-BOe18we8.js} +2 -2
- package/ui/dist/assets/SettingsWorkspace-Cdgnm4Wa.js +1 -0
- package/ui/dist/assets/{index-BhcvL0g-.js → index-C5PSahkr.js} +1 -1
- package/ui/dist/assets/index-Dn2AhL6d.css +1 -0
- package/ui/dist/assets/index-Tjqx6wUV.js +23 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/SettingsWorkspace-B3tgLrmF.js +0 -1
- package/ui/dist/assets/index-BtK3YhJc.js +0 -39
- package/ui/dist/assets/index-ClOLOjnA.css +0 -1
package/routes/video.js
CHANGED
|
@@ -2,7 +2,11 @@ import { mkdir, readFile, unlink, writeFile } from "fs/promises";
|
|
|
2
2
|
import { atomicWriteJson } from "../lib/atomicWrite.js";
|
|
3
3
|
import { join } from "path";
|
|
4
4
|
import { randomBytes } from "crypto";
|
|
5
|
-
import {
|
|
5
|
+
import { execFile } from "child_process";
|
|
6
|
+
import { tmpdir } from "os";
|
|
7
|
+
import { promisify } from "util";
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, setJobPhase, INFLIGHT_RETRY_AFTER_SECONDS } from "../lib/inflight.js";
|
|
6
10
|
import { isGenerationCanceledError, makeGenerationCanceledError } from "../lib/generationCancel.js";
|
|
7
11
|
import { logEvent, logError } from "../lib/logger.js";
|
|
8
12
|
import { invalidateHistoryIndex } from "../lib/historyIndex.js";
|
|
@@ -14,10 +18,22 @@ import { normalizeGrokVideoModel, normalizeVideoResolution, normalizeVideoAspect
|
|
|
14
18
|
import { errInfo } from "../lib/errInfo.js";
|
|
15
19
|
import { requireRuntimeContext } from "../lib/runtimeContext.js";
|
|
16
20
|
import { generateVideoThumbnail } from "../lib/videoThumb.js";
|
|
21
|
+
import { publish } from "../lib/eventBus.js";
|
|
22
|
+
import { publishJobEvent } from "../lib/ssePublish.js";
|
|
17
23
|
function sendSse(res, event, data) {
|
|
18
24
|
res.write(`event: ${event}\n`);
|
|
19
25
|
res.write(`data: ${JSON.stringify(data)}\n\n`);
|
|
20
26
|
}
|
|
27
|
+
function dualEmitVideo(res, requestId, event, data) {
|
|
28
|
+
if (!res.writableEnded)
|
|
29
|
+
sendSse(res, event, data);
|
|
30
|
+
if (event === "done") {
|
|
31
|
+
publishJobEvent(requestId, event, data);
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
publish(requestId, event, data);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
21
37
|
function toArray(v) {
|
|
22
38
|
return Array.isArray(v) ? v : [];
|
|
23
39
|
}
|
|
@@ -51,6 +67,32 @@ async function resolveSourceImage(ctx, sourceImage, sourceFilename) {
|
|
|
51
67
|
}
|
|
52
68
|
return { b64: null, filename: null };
|
|
53
69
|
}
|
|
70
|
+
const STORYBOARD_TRIM_SECONDS = "1.0";
|
|
71
|
+
async function trimStoryboardLeadIn(buffer, requestId) {
|
|
72
|
+
const tmpIn = join(tmpdir(), `ima2_sb_trim_in_${requestId.replace(/[^a-zA-Z0-9_-]/g, "_")}.mp4`);
|
|
73
|
+
const tmpOut = join(tmpdir(), `ima2_sb_trim_out_${requestId.replace(/[^a-zA-Z0-9_-]/g, "_")}.mp4`);
|
|
74
|
+
try {
|
|
75
|
+
await writeFile(tmpIn, buffer);
|
|
76
|
+
logEvent("video", "storyboard:trim-start", { requestId, inputBytes: buffer.length, trimSeconds: STORYBOARD_TRIM_SECONDS });
|
|
77
|
+
await execFileAsync("ffmpeg", [
|
|
78
|
+
"-y", "-ss", STORYBOARD_TRIM_SECONDS, "-i", tmpIn,
|
|
79
|
+
"-c:v", "libx264", "-preset", "fast", "-crf", "18",
|
|
80
|
+
"-c:a", "aac", "-b:a", "128k",
|
|
81
|
+
"-avoid_negative_ts", "make_zero", tmpOut,
|
|
82
|
+
], { timeout: 60_000 });
|
|
83
|
+
const trimmed = await readFile(tmpOut);
|
|
84
|
+
logEvent("video", "storyboard:trimmed", { requestId, originalBytes: buffer.length, trimmedBytes: trimmed.length, trimSeconds: STORYBOARD_TRIM_SECONDS });
|
|
85
|
+
return trimmed;
|
|
86
|
+
}
|
|
87
|
+
catch (trimError) {
|
|
88
|
+
logEvent("video", "storyboard:trim-exec-error", { requestId, error: trimError.message, stderr: trimError.stderr?.slice?.(0, 500) });
|
|
89
|
+
throw trimError;
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
await unlink(tmpIn).catch(() => { });
|
|
93
|
+
await unlink(tmpOut).catch(() => { });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
54
96
|
export function registerVideoRoutes(app, ctxRaw) {
|
|
55
97
|
const ctx = requireRuntimeContext(ctxRaw);
|
|
56
98
|
app.post("/api/video/generate", async (req, res) => {
|
|
@@ -59,22 +101,31 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
59
101
|
: typeof req.body?.clientRequestId === "string"
|
|
60
102
|
? req.body.clientRequestId
|
|
61
103
|
: req.id;
|
|
104
|
+
const asyncMode = req.body?.async === true;
|
|
62
105
|
let finishStatus = "completed";
|
|
63
106
|
let finishHttpStatus = 200;
|
|
64
107
|
let finishErrorCode;
|
|
65
108
|
let finishMeta = {};
|
|
66
109
|
let finishCanceled = false;
|
|
67
110
|
const cancelController = new AbortController();
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
111
|
+
if (!asyncMode) {
|
|
112
|
+
res.setHeader("Content-Type", "text/event-stream; charset=utf-8");
|
|
113
|
+
res.setHeader("Cache-Control", "no-cache, no-transform");
|
|
114
|
+
res.setHeader("Connection", "keep-alive");
|
|
115
|
+
res.flushHeaders?.();
|
|
116
|
+
}
|
|
72
117
|
const fail = (status, code, error, extra = {}) => {
|
|
73
118
|
const httpStatus = status ?? 500;
|
|
74
119
|
finishStatus = "error";
|
|
75
120
|
finishHttpStatus = httpStatus;
|
|
76
121
|
finishErrorCode = code;
|
|
77
|
-
|
|
122
|
+
const payload = { error, code, status: httpStatus, requestId, ...extra };
|
|
123
|
+
publish(requestId, "error", payload);
|
|
124
|
+
if (asyncMode && !res.headersSent) {
|
|
125
|
+
return res.status(httpStatus).json(payload);
|
|
126
|
+
}
|
|
127
|
+
if (!res.writableEnded)
|
|
128
|
+
sendSse(res, "error", payload);
|
|
78
129
|
};
|
|
79
130
|
try {
|
|
80
131
|
const { prompt, provider = "grok", model: rawModel } = req.body || {};
|
|
@@ -99,6 +150,14 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
99
150
|
"- Lock lighting direction, color palette, environment, and style.",
|
|
100
151
|
"- Describe ONLY what changes: action, camera movement, dialogue, sound.",
|
|
101
152
|
"",
|
|
153
|
+
"STORYBOARD IMAGE SOURCE RULE (HIGHEST PRIORITY — OVERRIDES ALL OTHER RULES):",
|
|
154
|
+
"- The source image is a 3x3 storyboard grid. Panel 1 (top-left) is a BLACK LEAD-IN FRAME — it contains no scene content.",
|
|
155
|
+
"- The video starts from black (Panel 1), then transitions into the action scene from Panel 2.",
|
|
156
|
+
"- Panels 2-9 contain the action sequence. Describe and animate only Panels 2-9.",
|
|
157
|
+
"- Start your rewritten prompt with: 'Fading in from black into the full-screen scene of [Panel 2 description],' — the server auto-trims the black lead-in.",
|
|
158
|
+
"- The storyboard grid must NEVER appear as a visible grid in any frame. The output is a single continuous cinematic clip.",
|
|
159
|
+
"- Do NOT reference Panel 1 in the action description — it is only a technical black frame.",
|
|
160
|
+
"",
|
|
102
161
|
"PROMPT STRUCTURE (layered caption format):",
|
|
103
162
|
"- Shot foundation: type + camera motion (dolly, pan, tracking, crane, static).",
|
|
104
163
|
"- Subject: action with intensity modifiers (crashes violently, drifts gently).",
|
|
@@ -165,25 +224,36 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
165
224
|
}
|
|
166
225
|
if (resolved.length > MAX_REF2V_REFERENCES)
|
|
167
226
|
return fail(400, "GROK_VIDEO_REF_TOO_MANY", `at most ${MAX_REF2V_REFERENCES} reference images`);
|
|
168
|
-
const
|
|
227
|
+
const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http") ? req.body.providerUrl : null;
|
|
228
|
+
const mode = incomingProviderUrl ? "image-to-video" : deriveVideoMode(resolved.length);
|
|
169
229
|
const duration = clampVideoDuration(durationCheck.duration, mode);
|
|
170
230
|
const referenceImages = mode === "reference-to-video" ? resolved.map((r) => r.b64) : undefined;
|
|
171
|
-
const sourceB64 = mode === "image-to-video" ? resolved[0]?.b64 : undefined;
|
|
231
|
+
const sourceB64 = incomingProviderUrl || (mode === "image-to-video" ? resolved[0]?.b64 : undefined);
|
|
172
232
|
const sourceFilename = resolved[0]?.filename ?? null;
|
|
173
|
-
startJob({
|
|
233
|
+
const started = startJob({
|
|
174
234
|
requestId,
|
|
175
235
|
kind: "video",
|
|
176
236
|
prompt: activePrompt,
|
|
177
237
|
meta: { kind: "video", sessionId, clientNodeId, model: modelCheck.model, mode, duration, resolution: resolutionCheck.resolution },
|
|
178
238
|
});
|
|
239
|
+
if (started && isStartJobFailure(started)) {
|
|
240
|
+
if (started.code === "TOO_MANY_JOBS") {
|
|
241
|
+
res.setHeader("Retry-After", String(INFLIGHT_RETRY_AFTER_SECONDS));
|
|
242
|
+
}
|
|
243
|
+
return fail(started.code === "TOO_MANY_JOBS" ? 429 : 409, started.code, started.code === "TOO_MANY_JOBS"
|
|
244
|
+
? "Too many concurrent generation jobs"
|
|
245
|
+
: "Request ID already in use");
|
|
246
|
+
}
|
|
179
247
|
registerJobAbortController(requestId, cancelController);
|
|
248
|
+
if (asyncMode)
|
|
249
|
+
res.status(202).json({ requestId });
|
|
180
250
|
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
181
251
|
logEvent("video", "request", { requestId, mode, duration, resolution: resolutionCheck.resolution, aspectRatio: aspectCheck.aspectRatio });
|
|
182
252
|
const startTime = Date.now();
|
|
183
253
|
const onEvent = (ev) => {
|
|
184
254
|
if (ev.phase === "submitted") {
|
|
185
255
|
setJobPhase(requestId, "streaming");
|
|
186
|
-
|
|
256
|
+
dualEmitVideo(res, requestId, "submitted", {
|
|
187
257
|
requestId,
|
|
188
258
|
xaiVideoRequestId: ev.xaiVideoRequestId,
|
|
189
259
|
requestedModel: ev.requestedModel,
|
|
@@ -192,11 +262,11 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
192
262
|
});
|
|
193
263
|
}
|
|
194
264
|
else if (ev.phase === "progress") {
|
|
195
|
-
|
|
265
|
+
dualEmitVideo(res, requestId, "progress", { requestId, progress: typeof ev.progress === "number" ? ev.progress / 100 : null, stalled: Boolean(ev.stalled) });
|
|
196
266
|
}
|
|
197
267
|
else {
|
|
198
268
|
setJobPhase(requestId, "planning");
|
|
199
|
-
|
|
269
|
+
dualEmitVideo(res, requestId, "planning", { requestId });
|
|
200
270
|
}
|
|
201
271
|
};
|
|
202
272
|
// Build prompt with series chain context
|
|
@@ -221,6 +291,7 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
221
291
|
plannerModel: plannerModel || undefined,
|
|
222
292
|
directApiKey,
|
|
223
293
|
onEvent,
|
|
294
|
+
storyboardActive,
|
|
224
295
|
});
|
|
225
296
|
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
226
297
|
const filename = `${Date.now()}_${rand}.mp4`;
|
|
@@ -234,6 +305,7 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
234
305
|
const meta = {
|
|
235
306
|
kind: "video",
|
|
236
307
|
mediaType: "video",
|
|
308
|
+
providerUrl: result.url,
|
|
237
309
|
requestId,
|
|
238
310
|
sessionId,
|
|
239
311
|
clientNodeId,
|
|
@@ -263,15 +335,25 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
263
335
|
...(topic ? { videoSeries: { topic, chainIndex: chain.length } } : {}),
|
|
264
336
|
...(storyboardActive ? { storyboard: true } : {}),
|
|
265
337
|
};
|
|
266
|
-
|
|
338
|
+
let finalBuffer = result.videoBuffer;
|
|
339
|
+
if (storyboardActive) {
|
|
340
|
+
try {
|
|
341
|
+
finalBuffer = await trimStoryboardLeadIn(result.videoBuffer, requestId);
|
|
342
|
+
}
|
|
343
|
+
catch (trimErr) {
|
|
344
|
+
logEvent("video", "storyboard:trim-failed", { requestId, error: trimErr.message });
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
await saveGeneratedVideoArtifact(ctx, filename, finalBuffer, meta);
|
|
267
348
|
generateVideoThumbnail(join(ctx.config.storage.generatedDir, filename)).catch(() => { });
|
|
268
349
|
invalidateHistoryIndex();
|
|
269
350
|
finishMeta = { filename, xaiVideoRequestId: result.xaiVideoRequestId };
|
|
270
351
|
logEvent("video", "saved", { requestId, filename, bytes: result.videoBuffer.length, elapsedMs: Date.now() - startTime });
|
|
271
|
-
|
|
352
|
+
dualEmitVideo(res, requestId, "done", {
|
|
272
353
|
requestId,
|
|
273
354
|
filename,
|
|
274
355
|
url: `/generated/${encodeURIComponent(filename)}`,
|
|
356
|
+
providerUrl: result.url,
|
|
275
357
|
mediaType: "video",
|
|
276
358
|
revisedPrompt: result.revisedPrompt,
|
|
277
359
|
elapsed,
|
|
@@ -291,19 +373,20 @@ export function registerVideoRoutes(app, ctxRaw) {
|
|
|
291
373
|
finishCanceled = true;
|
|
292
374
|
finishHttpStatus = canceled.status;
|
|
293
375
|
finishErrorCode = canceled.code;
|
|
294
|
-
|
|
376
|
+
dualEmitVideo(res, requestId, "error", { error: canceled.message, code: canceled.code, status: canceled.status, requestId });
|
|
295
377
|
}
|
|
296
378
|
else {
|
|
297
379
|
finishStatus = "error";
|
|
298
380
|
finishHttpStatus = err.status || 500;
|
|
299
381
|
finishErrorCode = err.code || "GROK_VIDEO_FAILED";
|
|
300
382
|
logError("video", "error", err.raw, { requestId, code: finishErrorCode });
|
|
301
|
-
|
|
383
|
+
dualEmitVideo(res, requestId, "error", { error: err.message, code: finishErrorCode, status: finishHttpStatus, requestId });
|
|
302
384
|
}
|
|
303
385
|
}
|
|
304
386
|
finally {
|
|
305
387
|
finishJob(requestId, { canceled: finishCanceled, status: finishStatus, httpStatus: finishHttpStatus, errorCode: finishErrorCode, meta: finishMeta });
|
|
306
|
-
res.
|
|
388
|
+
if (!res.writableEnded)
|
|
389
|
+
res.end();
|
|
307
390
|
}
|
|
308
391
|
});
|
|
309
392
|
}
|
package/skills/ima2/SKILL.md
CHANGED
|
@@ -143,13 +143,22 @@ Do not use positional edit prompts. `ima2 edit` requires `--prompt`.
|
|
|
143
143
|
|
|
144
144
|
## Parallel Generation
|
|
145
145
|
|
|
146
|
-
There is no `--parallel` flag. For
|
|
146
|
+
There is no `--parallel` flag. For multiple candidates from the same prompt,
|
|
147
|
+
prefer one server-side batch request:
|
|
147
148
|
|
|
148
149
|
```bash
|
|
149
|
-
ima2 gen "
|
|
150
|
-
ima2
|
|
151
|
-
|
|
152
|
-
|
|
150
|
+
ima2 gen "four poster candidates" -n 4 -d ./out --quality high
|
|
151
|
+
ima2 multimode "four different poster directions" --max-images 4
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
For truly different prompts, independent CLI jobs can run concurrently against
|
|
155
|
+
the same server. Capture request IDs with JSON output, then monitor or cancel:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
ima2 gen "variation 1" --quality high --json
|
|
159
|
+
ima2 gen "variation 2" --quality high --json
|
|
160
|
+
ima2 ps --json
|
|
161
|
+
ima2 cancel <requestId>
|
|
153
162
|
```
|
|
154
163
|
|
|
155
164
|
Treat `capabilities.limits.maxParallel` as advisory client-side queue guidance only.
|
|
@@ -402,6 +411,12 @@ Generate a high-quality still image first, then animate it. This produces better
|
|
|
402
411
|
|
|
403
412
|
**Critical rule for i2v**: Compose ALL characters and the environment together in ONE image. Do NOT use individual portrait refs for i2v — the video model needs a single composed scene to animate from.
|
|
404
413
|
|
|
414
|
+
**Keyframe image provider rule (MANDATORY)**:
|
|
415
|
+
- **Primary**: GPT Image 2 (OpenAI, `provider: oauth`) with `quality: high`, maximum resolution matching the target video aspect ratio. For 16:9 video use `1792x1024`. For 1:1 use `1024x1024`. For 9:16 use `1024x1792`.
|
|
416
|
+
- **Fallback**: Grok (`provider: grok`, model `grok-imagine-image-quality`). Only aspect ratio must match — resolution does not matter because i2v accepts any resolution source image and internally rescales.
|
|
417
|
+
- GPT Image 2 produces superior keyframes: better lighting coherence, character consistency, and fine detail that survives i2v animation. Always try GPT first.
|
|
418
|
+
- The i2v model internally rescales the source image to its native resolution regardless of input size, so there is no benefit to upscaling a Grok fallback image.
|
|
419
|
+
|
|
405
420
|
**ref2v vs i2v decision**:
|
|
406
421
|
|
|
407
422
|
| Scenario | Use | Why |
|
|
@@ -412,8 +427,12 @@ Generate a high-quality still image first, then animate it. This produces better
|
|
|
412
427
|
|
|
413
428
|
```bash
|
|
414
429
|
# Multi-character scene: compose BOTH characters in one image first
|
|
430
|
+
# Primary: GPT Image 2 at high quality, max resolution, aspect ratio matching 16:9 video
|
|
415
431
|
ima2 gen "cinematic wide shot of Bruce Lee in yellow tracksuit facing Elon Musk in dark gi, underground fight arena, dramatic lighting, 16:9" --quality high --size 1792x1024 -o scene.png
|
|
416
432
|
|
|
433
|
+
# Fallback if GPT fails: Grok quality model, match aspect ratio only
|
|
434
|
+
# ima2 gen "same prompt" --provider grok --model grok-imagine-image-quality --size 1824x1024 -o scene.png
|
|
435
|
+
|
|
417
436
|
# Then animate from the composed scene
|
|
418
437
|
ima2 video "Bruce throws a rapid jeet kune do combination" --ref scene.png --duration 10 --resolution 720p --aspect-ratio 16:9
|
|
419
438
|
```
|
|
@@ -438,30 +457,88 @@ ima2 video "close-up of rain drops on a neon sign reflection" \
|
|
|
438
457
|
|
|
439
458
|
The planner receives previous prompts from the same topic as continuity context. This is best-effort prompt guidance, not a guarantee that subjects, palette, or style will remain identical. For branch-local continuation, use `ima2 video continue` instead.
|
|
440
459
|
|
|
441
|
-
#### Storyboard-to-Video Chaining (
|
|
460
|
+
#### Storyboard-to-Video Chaining (9-panel storyboard → i2v loop)
|
|
461
|
+
|
|
462
|
+
The highest-quality video production workflow. Since Grok i2v accepts only **one image input**, pack the entire action sequence into a single 3×3 (9-panel) storyboard grid image. The i2v model reads the panels as a visual script and animates the progression.
|
|
463
|
+
|
|
464
|
+
**Full workflow**:
|
|
465
|
+
|
|
466
|
+
```
|
|
467
|
+
keyframe image (GPT high)
|
|
468
|
+
→ GPT i2i with reference → 9-panel storyboard grid
|
|
469
|
+
→ Grok i2v (reads panels, animates sequence)
|
|
470
|
+
→ extract last frame
|
|
471
|
+
→ GPT i2i with last frame → next 9-panel storyboard
|
|
472
|
+
→ Grok i2v
|
|
473
|
+
→ repeat
|
|
474
|
+
```
|
|
475
|
+
|
|
476
|
+
**Step 1 — Opening keyframe** (GPT Image 2, `quality: high`, max resolution matching target aspect ratio):
|
|
477
|
+
|
|
478
|
+
```bash
|
|
479
|
+
ima2 gen "cinematic wide shot of two fighters in a dojo, dramatic lighting" \
|
|
480
|
+
--quality high --size 1792x1024 --storyboard
|
|
481
|
+
```
|
|
482
|
+
|
|
483
|
+
Fallback: Grok `grok-imagine-image-quality`, match aspect ratio only — resolution does not matter because i2v internally rescales.
|
|
442
484
|
|
|
443
|
-
|
|
485
|
+
**Step 2 — 9-panel storyboard grid** (GPT Image 2 with keyframe as reference):
|
|
444
486
|
|
|
445
487
|
```bash
|
|
446
|
-
#
|
|
447
|
-
ima2 gen "
|
|
488
|
+
# Use the keyframe as reference, prompt describes 9 sequential panels
|
|
489
|
+
ima2 gen "Using this scene as reference, create a 3x3 storyboard grid (9 panels, thin black borders) showing a 15-second action sequence. Panel 1 (0s): ... Panel 2 (2s): ... Panel 9 (15s): ... Maintain identical character designs across all panels." \
|
|
490
|
+
--ref keyframe.png --quality high --size 1024x1024
|
|
491
|
+
```
|
|
448
492
|
|
|
449
|
-
|
|
450
|
-
|
|
493
|
+
**9-panel storyboard rules**:
|
|
494
|
+
- Grid layout: 3×3, thin black borders between panels
|
|
495
|
+
- Read order: left-to-right, top-to-bottom (panels 1-9)
|
|
496
|
+
- **Panel 1 (top-left) MUST be solid black** — this is a lead-in frame, not content. The i2v model starts from Panel 1's pixels; a black frame ensures the video begins with a clean fade-in instead of showing the grid. The 1-second black lead-in is auto-trimmed by the server.
|
|
497
|
+
- Panels 2-9 carry the action sequence (8 key moments with timestamps)
|
|
498
|
+
- Character designs MUST be identical across all panels
|
|
499
|
+
- Vary camera angle per panel for dynamic energy
|
|
500
|
+
- Each panel should look like a film still, not a sketch
|
|
501
|
+
- Do NOT add timestamp labels or text to panels — they burn into the video
|
|
502
|
+
- Square format (1024×1024) works best — i2v rescales internally
|
|
451
503
|
|
|
452
|
-
|
|
453
|
-
CLIP1=$(ima2 ls -n 1 --json | jq -r '.items[0].filename')
|
|
454
|
-
ima2 video continue "Elon counterattacks with haymaker" --video "$CLIP1" --duration 10
|
|
504
|
+
**Step 3 — Animate storyboard via i2v**:
|
|
455
505
|
|
|
456
|
-
|
|
506
|
+
```bash
|
|
507
|
+
ima2 video "This is a 9-panel storyboard. Animate the full sequence as one continuous 15-second clip following panels left-to-right, top-to-bottom. Panel 1: ... Panel 9: ... Sound: [describe music, SFX, dialogue]. Camera: [describe movement per beat]." \
|
|
508
|
+
--ref storyboard.png --duration 15 --resolution 720p --model grok-imagine-video-1.5-preview
|
|
457
509
|
```
|
|
458
510
|
|
|
459
|
-
**
|
|
460
|
-
-
|
|
461
|
-
-
|
|
462
|
-
-
|
|
463
|
-
-
|
|
464
|
-
-
|
|
511
|
+
**i2v prompt rules for storyboard input**:
|
|
512
|
+
- Explicitly state "This is a 9-panel storyboard" at the start
|
|
513
|
+
- Reference each panel by number with its action description
|
|
514
|
+
- Always include Sound/Music direction — never leave audio undefined
|
|
515
|
+
- Include Camera direction per beat (wide, close-up, tracking, handheld, slow-mo)
|
|
516
|
+
- Describe the end frame explicitly for continuation
|
|
517
|
+
|
|
518
|
+
**Step 4 — Extract last frame and repeat**:
|
|
519
|
+
|
|
520
|
+
```bash
|
|
521
|
+
# Extract last frame via ffmpeg
|
|
522
|
+
ffmpeg -sseof -0.1 -i clip.mp4 -frames:v 1 -q:v 2 -update 1 lastframe.jpg -y
|
|
523
|
+
|
|
524
|
+
# Generate next storyboard using last frame as reference
|
|
525
|
+
ima2 gen "Using this fight scene last frame as reference, create a 3x3 storyboard grid..." \
|
|
526
|
+
--ref lastframe.jpg --quality high --size 1024x1024
|
|
527
|
+
|
|
528
|
+
# Animate next storyboard
|
|
529
|
+
ima2 video "This is a 9-panel storyboard..." --ref storyboard2.png --duration 15
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
**Fallback: continueFromVideo** — If a storyboard image triggers content moderation (common with intense action/fight scenes), fall back to `video continue` with a detailed text prompt instead:
|
|
533
|
+
|
|
534
|
+
```bash
|
|
535
|
+
ima2 video continue "detailed action description with sound and camera direction" \
|
|
536
|
+
--video "$PREV_CLIP" --duration 15
|
|
537
|
+
```
|
|
538
|
+
|
|
539
|
+
**Clip duration is flexible** — use 15s for action-dense sequences with many beats, 10s for transitions, 5s for quick cuts. The 9-panel storyboard works best with 15s clips (each panel ≈ 1.5-2s of screen time).
|
|
540
|
+
|
|
541
|
+
**Music and sound are MANDATORY** in i2v prompts — describe the score (orchestral, percussion, taiko drums), sound effects (impacts, whooshes, crashes), dialogue lines, and audio transitions. "No music" or undefined audio produces flat, lifeless output.
|
|
465
542
|
|
|
466
543
|
#### Video Continuation (extend/sequel)
|
|
467
544
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"index.html": {
|
|
3
|
-
"file": "assets/index-
|
|
3
|
+
"file": "assets/index-Tjqx6wUV.js",
|
|
4
4
|
"name": "index",
|
|
5
5
|
"src": "index.html",
|
|
6
6
|
"isEntry": true,
|
|
@@ -16,11 +16,11 @@
|
|
|
16
16
|
"src/components/PromptLibraryPanel.tsx"
|
|
17
17
|
],
|
|
18
18
|
"css": [
|
|
19
|
-
"assets/index-
|
|
19
|
+
"assets/index-Dn2AhL6d.css"
|
|
20
20
|
]
|
|
21
21
|
},
|
|
22
22
|
"src/components/NodeCanvas.tsx": {
|
|
23
|
-
"file": "assets/NodeCanvas-
|
|
23
|
+
"file": "assets/NodeCanvas-C4ifFzB1.js",
|
|
24
24
|
"name": "NodeCanvas",
|
|
25
25
|
"src": "src/components/NodeCanvas.tsx",
|
|
26
26
|
"isDynamicEntry": true,
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
]
|
|
33
33
|
},
|
|
34
34
|
"src/components/PromptImportDialog.tsx": {
|
|
35
|
-
"file": "assets/PromptImportDialog-
|
|
35
|
+
"file": "assets/PromptImportDialog-CgQ94Gth.js",
|
|
36
36
|
"name": "PromptImportDialog",
|
|
37
37
|
"src": "src/components/PromptImportDialog.tsx",
|
|
38
38
|
"isDynamicEntry": true,
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
]
|
|
46
46
|
},
|
|
47
47
|
"src/components/PromptImportDiscoverySection.tsx": {
|
|
48
|
-
"file": "assets/PromptImportDiscoverySection-
|
|
48
|
+
"file": "assets/PromptImportDiscoverySection-CuzyzbNI.js",
|
|
49
49
|
"name": "PromptImportDiscoverySection",
|
|
50
50
|
"src": "src/components/PromptImportDiscoverySection.tsx",
|
|
51
51
|
"isDynamicEntry": true,
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
]
|
|
55
55
|
},
|
|
56
56
|
"src/components/PromptImportFolderSection.tsx": {
|
|
57
|
-
"file": "assets/PromptImportFolderSection-
|
|
57
|
+
"file": "assets/PromptImportFolderSection-DHLGlO6l.js",
|
|
58
58
|
"name": "PromptImportFolderSection",
|
|
59
59
|
"src": "src/components/PromptImportFolderSection.tsx",
|
|
60
60
|
"isDynamicEntry": true,
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
]
|
|
64
64
|
},
|
|
65
65
|
"src/components/PromptLibraryPanel.tsx": {
|
|
66
|
-
"file": "assets/PromptLibraryPanel-
|
|
66
|
+
"file": "assets/PromptLibraryPanel-BOe18we8.js",
|
|
67
67
|
"name": "PromptLibraryPanel",
|
|
68
68
|
"src": "src/components/PromptLibraryPanel.tsx",
|
|
69
69
|
"isDynamicEntry": true,
|
|
@@ -75,7 +75,7 @@
|
|
|
75
75
|
]
|
|
76
76
|
},
|
|
77
77
|
"src/components/SettingsWorkspace.tsx": {
|
|
78
|
-
"file": "assets/SettingsWorkspace-
|
|
78
|
+
"file": "assets/SettingsWorkspace-Cdgnm4Wa.js",
|
|
79
79
|
"name": "SettingsWorkspace",
|
|
80
80
|
"src": "src/components/SettingsWorkspace.tsx",
|
|
81
81
|
"isDynamicEntry": true,
|
|
@@ -84,7 +84,7 @@
|
|
|
84
84
|
]
|
|
85
85
|
},
|
|
86
86
|
"src/components/agent/AgentWorkspace.tsx": {
|
|
87
|
-
"file": "assets/AgentWorkspace-
|
|
87
|
+
"file": "assets/AgentWorkspace-Dth6YijN.js",
|
|
88
88
|
"name": "AgentWorkspace",
|
|
89
89
|
"src": "src/components/agent/AgentWorkspace.tsx",
|
|
90
90
|
"isDynamicEntry": true,
|
|
@@ -93,7 +93,7 @@
|
|
|
93
93
|
]
|
|
94
94
|
},
|
|
95
95
|
"src/components/canvas-mode/index.ts": {
|
|
96
|
-
"file": "assets/index-
|
|
96
|
+
"file": "assets/index-C5PSahkr.js",
|
|
97
97
|
"name": "index",
|
|
98
98
|
"src": "src/components/canvas-mode/index.ts",
|
|
99
99
|
"isDynamicEntry": true,
|
|
@@ -102,7 +102,7 @@
|
|
|
102
102
|
]
|
|
103
103
|
},
|
|
104
104
|
"src/components/card-news/CardNewsWorkspace.tsx": {
|
|
105
|
-
"file": "assets/CardNewsWorkspace-
|
|
105
|
+
"file": "assets/CardNewsWorkspace-Dav3K5CT.js",
|
|
106
106
|
"name": "CardNewsWorkspace",
|
|
107
107
|
"src": "src/components/card-news/CardNewsWorkspace.tsx",
|
|
108
108
|
"isDynamicEntry": true,
|
|
@@ -111,7 +111,7 @@
|
|
|
111
111
|
]
|
|
112
112
|
},
|
|
113
113
|
"src/components/prompt-builder/PromptBuilderPanel.tsx": {
|
|
114
|
-
"file": "assets/PromptBuilderPanel-
|
|
114
|
+
"file": "assets/PromptBuilderPanel-CEcyU9PL.js",
|
|
115
115
|
"name": "PromptBuilderPanel",
|
|
116
116
|
"src": "src/components/prompt-builder/PromptBuilderPanel.tsx",
|
|
117
117
|
"isDynamicEntry": true,
|