ima2-gen 1.1.19 → 1.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -25
- package/bin/commands/capabilities.js +2 -2
- package/bin/commands/capabilities.ts +2 -2
- package/bin/commands/defaults.js +2 -2
- package/bin/commands/defaults.ts +2 -2
- package/bin/commands/doctor.js +3 -3
- package/bin/commands/doctor.ts +3 -3
- package/bin/commands/edit.js +1 -1
- package/bin/commands/edit.ts +1 -1
- package/bin/commands/gen.js +1 -1
- package/bin/commands/gen.ts +1 -1
- package/bin/commands/grok.js +25 -22
- package/bin/commands/grok.ts +26 -22
- package/bin/commands/multimode.js +1 -1
- package/bin/commands/multimode.ts +1 -1
- package/bin/commands/observability.js +2 -2
- package/bin/commands/observability.ts +2 -2
- package/bin/commands/video.js +335 -13
- package/bin/commands/video.ts +249 -12
- package/bin/ima2.js +9 -9
- package/bin/ima2.ts +9 -9
- package/bin/lib/error-hints.js +2 -2
- package/bin/lib/error-hints.ts +2 -2
- package/docs/API.md +112 -3
- package/docs/CLI.md +61 -7
- package/docs/FAQ.ko.md +15 -20
- package/docs/FAQ.md +14 -19
- package/docs/NPX_QUICKSTART.md +40 -0
- package/docs/PROMPT_STUDIO.ko.md +1 -1
- package/docs/PROMPT_STUDIO.md +1 -1
- package/docs/README.ja.md +6 -16
- package/docs/README.ko.md +10 -20
- package/docs/README.zh-CN.md +7 -17
- package/docs/migration/runtime-test-inventory.md +9 -1
- package/lib/agentGenerationPlanner.js +20 -1
- package/lib/agentGenerationPlanner.ts +25 -1
- package/lib/agentRuntime.js +24 -8
- package/lib/agentRuntime.ts +23 -8
- package/lib/capabilities.js +1 -1
- package/lib/capabilities.ts +1 -1
- package/lib/generationErrors.js +1 -1
- package/lib/generationErrors.ts +1 -1
- package/lib/grokProxyLauncher.js +26 -3
- package/lib/grokProxyLauncher.ts +27 -3
- package/lib/grokVideoAdapter.js +18 -89
- package/lib/grokVideoAdapter.ts +27 -88
- package/lib/grokVideoCanvas.js +25 -0
- package/lib/grokVideoCanvas.ts +26 -0
- package/lib/grokVideoDownload.js +58 -0
- package/lib/grokVideoDownload.ts +59 -0
- package/lib/grokVideoPlannerPrompt.js +64 -0
- package/lib/grokVideoPlannerPrompt.ts +67 -0
- package/lib/historyList.js +7 -1
- package/lib/historyList.ts +5 -1
- package/lib/oauthLauncher.js +21 -6
- package/lib/oauthLauncher.ts +22 -6
- package/lib/videoContinuity.js +149 -0
- package/lib/videoContinuity.ts +180 -0
- package/lib/videoFrameExtract.js +80 -0
- package/lib/videoFrameExtract.ts +78 -0
- package/node_modules/progrok/dist/index.js +187 -88
- package/node_modules/progrok/dist/index.js.map +1 -1
- package/node_modules/progrok/package.json +1 -1
- package/node_modules/progrok/skills/progrok/SKILL.md +33 -4
- package/package.json +2 -2
- package/routes/index.js +4 -0
- package/routes/index.ts +4 -0
- package/routes/quota.js +66 -0
- package/routes/quota.ts +89 -0
- package/routes/video.js +77 -15
- package/routes/video.ts +82 -14
- package/routes/videoExtended.js +293 -0
- package/routes/videoExtended.ts +284 -0
- package/server.js +6 -2
- package/server.ts +5 -2
- package/skills/ima2/SKILL.md +381 -3
- package/ui/dist/.vite/manifest.json +12 -12
- package/ui/dist/assets/{AgentWorkspace-DE_wg90f.js → AgentWorkspace-B_hq9CLg.js} +2 -2
- package/ui/dist/assets/{CardNewsWorkspace--Myc5pAp.js → CardNewsWorkspace-wD12J7qk.js} +1 -1
- package/ui/dist/assets/{NodeCanvas-4U5oOT2y.js → NodeCanvas-CI_wuPMf.js} +1 -1
- package/ui/dist/assets/{PromptBuilderPanel-DNW1U8zI.js → PromptBuilderPanel-CUTujJUV.js} +1 -1
- package/ui/dist/assets/{PromptImportDialog-o-4Sqki1.js → PromptImportDialog-CUi66jPK.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-BAbrRP8B.js → PromptImportDiscoverySection-Cm3vrjY4.js} +1 -1
- package/ui/dist/assets/{PromptImportFolderSection-L-XI2noz.js → PromptImportFolderSection-DOtWTD9n.js} +1 -1
- package/ui/dist/assets/{PromptLibraryPanel-CrW9LYGD.js → PromptLibraryPanel-BMjQegRa.js} +2 -2
- package/ui/dist/assets/SettingsWorkspace-PiaVnsdA.js +1 -0
- package/ui/dist/assets/{index-BONbNNIi.js → index-31uVIdt4.js} +1 -1
- package/ui/dist/assets/index-CjgnNtgt.css +1 -0
- package/ui/dist/assets/index-Da2s4_-5.js +36 -0
- package/ui/dist/index.html +2 -2
- package/vendor/progrok-0.2.0.tgz +0 -0
- package/ui/dist/assets/SettingsWorkspace-Dn4SYTyZ.js +0 -1
- package/ui/dist/assets/index-B6tcw_UF.css +0 -1
- package/ui/dist/assets/index-CeSZ2L3-.js +0 -32
- package/vendor/progrok-0.1.1.tgz +0 -0
package/routes/video.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
1
|
+
import { mkdir, readFile, unlink, writeFile } from "fs/promises";
|
|
2
2
|
import { join } from "path";
|
|
3
3
|
import { randomBytes } from "crypto";
|
|
4
4
|
import type { Express, Request, Response } from "express";
|
|
@@ -8,6 +8,17 @@ import { logEvent, logError } from "../lib/logger.js";
|
|
|
8
8
|
import { invalidateHistoryIndex } from "../lib/historyIndex.js";
|
|
9
9
|
import { generateVideoViaGrok, type GrokVideoEvent } from "../lib/grokVideoAdapter.js";
|
|
10
10
|
import { getVideoSeriesChain } from "../lib/videoSeriesChain.js";
|
|
11
|
+
import {
|
|
12
|
+
ACTIVE_VIDEO_PROMPT_GUIDANCE,
|
|
13
|
+
appendVideoContinuityEntry,
|
|
14
|
+
lineageFromVideoMetadata,
|
|
15
|
+
normalizeVideoContinuityLineage,
|
|
16
|
+
readVideoSidecar,
|
|
17
|
+
requireActiveVideoPrompt,
|
|
18
|
+
safeGeneratedVideoFilename,
|
|
19
|
+
type VideoContinuityLineage,
|
|
20
|
+
} from "../lib/videoContinuity.js";
|
|
21
|
+
import { extractGeneratedVideoFrameB64 } from "../lib/videoFrameExtract.js";
|
|
11
22
|
import {
|
|
12
23
|
normalizeGrokVideoModel,
|
|
13
24
|
normalizeVideoResolution,
|
|
@@ -36,6 +47,17 @@ function isNormalizeError(x: unknown): x is NormalizeError {
|
|
|
36
47
|
return typeof x === "object" && x !== null && typeof (x as { error?: unknown }).error === "string";
|
|
37
48
|
}
|
|
38
49
|
|
|
50
|
+
export async function saveGeneratedVideoArtifact(ctx: RuntimeContext, filename: string, buffer: Buffer, metadata: unknown): Promise<void> {
|
|
51
|
+
const filePath = join(ctx.config.storage.generatedDir, filename);
|
|
52
|
+
await writeFile(filePath, buffer);
|
|
53
|
+
try {
|
|
54
|
+
await writeFile(`${filePath}.json`, JSON.stringify(metadata));
|
|
55
|
+
} catch (err) {
|
|
56
|
+
await unlink(filePath).catch(() => {});
|
|
57
|
+
throw err;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
39
61
|
async function resolveSourceImage(
|
|
40
62
|
ctx: RuntimeContext,
|
|
41
63
|
sourceImage: unknown,
|
|
@@ -44,6 +66,7 @@ async function resolveSourceImage(
|
|
|
44
66
|
if (typeof sourceFilename === "string" && sourceFilename) {
|
|
45
67
|
const safe = sourceFilename.replace(/^\/+/, "");
|
|
46
68
|
if (safe.includes("..")) throw { status: 400, code: "GROK_VIDEO_INVALID_MODE", message: "invalid source filename" };
|
|
69
|
+
if (/\.mp4$/i.test(safe)) throw { status: 400, code: "GROK_VIDEO_INVALID_MODE", message: "use continueFromVideo for generated video continuation" };
|
|
47
70
|
const buf = await readFile(join(ctx.config.storage.generatedDir, safe));
|
|
48
71
|
return { b64: buf.toString("base64"), filename: safe };
|
|
49
72
|
}
|
|
@@ -74,12 +97,12 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
74
97
|
res.setHeader("Connection", "keep-alive");
|
|
75
98
|
res.flushHeaders?.();
|
|
76
99
|
|
|
77
|
-
const fail = (status: number | undefined, code: string, error: string) => {
|
|
100
|
+
const fail = (status: number | undefined, code: string, error: string, extra: Record<string, unknown> = {}) => {
|
|
78
101
|
const httpStatus = status ?? 500;
|
|
79
102
|
finishStatus = "error";
|
|
80
103
|
finishHttpStatus = httpStatus;
|
|
81
104
|
finishErrorCode = code;
|
|
82
|
-
sendSse(res, "error", { error, code, status: httpStatus, requestId });
|
|
105
|
+
sendSse(res, "error", { error, code, status: httpStatus, requestId, ...extra });
|
|
83
106
|
};
|
|
84
107
|
|
|
85
108
|
try {
|
|
@@ -89,7 +112,8 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
89
112
|
const topic = typeof req.body?.topic === "string" ? req.body.topic.trim() : "";
|
|
90
113
|
|
|
91
114
|
if (provider !== "grok") return fail(400, "VIDEO_PROVIDER_UNSUPPORTED", "video generation requires provider 'grok'");
|
|
92
|
-
|
|
115
|
+
const activePrompt = requireActiveVideoPrompt(prompt);
|
|
116
|
+
if (!activePrompt) return fail(400, "PROMPT_REQUIRED", "Prompt is required", { guidance: ACTIVE_VIDEO_PROMPT_GUIDANCE });
|
|
93
117
|
|
|
94
118
|
const modelCheck = normalizeGrokVideoModel(rawModel);
|
|
95
119
|
if (isNormalizeError(modelCheck)) return fail(modelCheck.status, modelCheck.code, modelCheck.error);
|
|
@@ -101,6 +125,20 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
101
125
|
if (isNormalizeError(aspectCheck)) return fail(aspectCheck.status, aspectCheck.code, aspectCheck.error);
|
|
102
126
|
|
|
103
127
|
// Resolve reference inputs: base64 list + existing-file list + legacy single source.
|
|
128
|
+
let parentLineage: VideoContinuityLineage | null = null;
|
|
129
|
+
let continueFromVideoFilename: string | null = null;
|
|
130
|
+
if (typeof req.body?.continueFromVideo === "string" && req.body.continueFromVideo.trim()) {
|
|
131
|
+
try {
|
|
132
|
+
continueFromVideoFilename = safeGeneratedVideoFilename(req.body.continueFromVideo);
|
|
133
|
+
const parentMeta = await readVideoSidecar(ctx.config.storage.generatedDir, continueFromVideoFilename);
|
|
134
|
+
parentLineage = lineageFromVideoMetadata(continueFromVideoFilename, parentMeta);
|
|
135
|
+
} catch (e: any) {
|
|
136
|
+
return fail(e?.status || 400, "GROK_VIDEO_INVALID_MODE", e?.message || "invalid continuation video");
|
|
137
|
+
}
|
|
138
|
+
} else {
|
|
139
|
+
parentLineage = normalizeVideoContinuityLineage(req.body?.continuityLineage);
|
|
140
|
+
}
|
|
141
|
+
|
|
104
142
|
const refInputs: Array<{ image?: unknown; filename?: unknown }> = [
|
|
105
143
|
...toArray(req.body?.referenceImages).map((image) => ({ image })),
|
|
106
144
|
...toArray(req.body?.referenceFilenames).map((filename) => ({ filename })),
|
|
@@ -108,6 +146,13 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
108
146
|
? [{ image: req.body?.sourceImage, filename: req.body?.sourceFilename }]
|
|
109
147
|
: []),
|
|
110
148
|
];
|
|
149
|
+
if (continueFromVideoFilename && !req.body?.sourceImage && !req.body?.sourceFilename) {
|
|
150
|
+
try {
|
|
151
|
+
refInputs.push({ image: await extractGeneratedVideoFrameB64(ctx.config.storage.generatedDir, continueFromVideoFilename) });
|
|
152
|
+
} catch (e: any) {
|
|
153
|
+
return fail(e?.status || 500, "GROK_VIDEO_FRAME_FAILED", e?.message || "failed to extract continuation frame");
|
|
154
|
+
}
|
|
155
|
+
}
|
|
111
156
|
let resolved: Array<{ b64: string; filename: string | null }>;
|
|
112
157
|
try {
|
|
113
158
|
const all = await Promise.all(refInputs.map((r) => resolveSourceImage(ctx, r.image, r.filename)));
|
|
@@ -125,7 +170,7 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
125
170
|
startJob({
|
|
126
171
|
requestId,
|
|
127
172
|
kind: "video",
|
|
128
|
-
prompt,
|
|
173
|
+
prompt: activePrompt,
|
|
129
174
|
meta: { kind: "video", sessionId, clientNodeId, model: modelCheck.model, mode, duration, resolution: resolutionCheck.resolution },
|
|
130
175
|
});
|
|
131
176
|
registerJobAbortController(requestId, cancelController);
|
|
@@ -137,7 +182,13 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
137
182
|
const onEvent = (ev: GrokVideoEvent) => {
|
|
138
183
|
if (ev.phase === "submitted") {
|
|
139
184
|
setJobPhase(requestId, "streaming");
|
|
140
|
-
sendSse(res, "submitted", {
|
|
185
|
+
sendSse(res, "submitted", {
|
|
186
|
+
requestId,
|
|
187
|
+
xaiVideoRequestId: ev.xaiVideoRequestId,
|
|
188
|
+
requestedModel: ev.requestedModel,
|
|
189
|
+
effectiveModel: ev.effectiveModel,
|
|
190
|
+
modelFallback: ev.modelFallback ?? null,
|
|
191
|
+
});
|
|
141
192
|
} else if (ev.phase === "progress") {
|
|
142
193
|
sendSse(res, "progress", { requestId, progress: typeof ev.progress === "number" ? ev.progress / 100 : null, stalled: Boolean(ev.stalled) });
|
|
143
194
|
} else {
|
|
@@ -147,10 +198,10 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
147
198
|
};
|
|
148
199
|
|
|
149
200
|
// Build prompt with series chain context
|
|
150
|
-
const chain = topic ? await getVideoSeriesChain(ctx.config.storage.generatedDir, topic) : [];
|
|
201
|
+
const chain = !parentLineage && topic ? await getVideoSeriesChain(ctx.config.storage.generatedDir, topic) : [];
|
|
151
202
|
const effectivePrompt = chain.length > 0
|
|
152
|
-
? `[Series topic: ${topic}]\n[Previous prompts in series:\n${chain.map((p, i) => `${i + 1}. ${p}`).join("\n")}\n]\n\n${
|
|
153
|
-
:
|
|
203
|
+
? `[Series topic: ${topic}]\n[Previous prompts in series:\n${chain.map((p, i) => `${i + 1}. ${p}`).join("\n")}\n]\n\n${activePrompt}`
|
|
204
|
+
: activePrompt;
|
|
154
205
|
|
|
155
206
|
const result = await generateVideoViaGrok(effectivePrompt, ctx, {
|
|
156
207
|
model: modelCheck.model,
|
|
@@ -162,23 +213,33 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
162
213
|
referenceImages,
|
|
163
214
|
signal: cancelController.signal,
|
|
164
215
|
requestId,
|
|
216
|
+
continuityLineage: parentLineage,
|
|
165
217
|
onEvent,
|
|
166
218
|
});
|
|
167
219
|
|
|
168
220
|
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
169
221
|
const filename = `${Date.now()}_${rand}.mp4`;
|
|
170
222
|
const elapsed = +((Date.now() - startTime) / 1000).toFixed(1);
|
|
223
|
+
const videoContinuity = appendVideoContinuityEntry(parentLineage, {
|
|
224
|
+
filename,
|
|
225
|
+
userPrompt: activePrompt,
|
|
226
|
+
revisedPrompt: result.revisedPrompt,
|
|
227
|
+
createdAt: Date.now(),
|
|
228
|
+
});
|
|
171
229
|
const meta = {
|
|
172
230
|
kind: "video",
|
|
173
231
|
mediaType: "video",
|
|
174
232
|
requestId,
|
|
175
233
|
sessionId,
|
|
176
234
|
clientNodeId,
|
|
177
|
-
prompt,
|
|
178
|
-
userPrompt:
|
|
235
|
+
prompt: activePrompt,
|
|
236
|
+
userPrompt: activePrompt,
|
|
179
237
|
revisedPrompt: result.revisedPrompt,
|
|
180
238
|
provider: "grok",
|
|
181
|
-
model:
|
|
239
|
+
model: result.effectiveModel,
|
|
240
|
+
requestedModel: result.requestedModel,
|
|
241
|
+
effectiveModel: result.effectiveModel,
|
|
242
|
+
modelFallback: result.modelFallback,
|
|
182
243
|
createdAt: Date.now(),
|
|
183
244
|
elapsed,
|
|
184
245
|
usage: result.usage,
|
|
@@ -189,11 +250,14 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
189
250
|
aspectRatio: result.aspectRatio,
|
|
190
251
|
sourceImageFilename: sourceFilename,
|
|
191
252
|
xaiVideoRequestId: result.xaiVideoRequestId,
|
|
253
|
+
requestedModel: result.requestedModel,
|
|
254
|
+
effectiveModel: result.effectiveModel,
|
|
255
|
+
modelFallback: result.modelFallback,
|
|
192
256
|
},
|
|
257
|
+
videoContinuity,
|
|
193
258
|
...(topic ? { videoSeries: { topic, chainIndex: chain.length } } : {}),
|
|
194
259
|
};
|
|
195
|
-
await
|
|
196
|
-
await writeFile(join(ctx.config.storage.generatedDir, filename + ".json"), JSON.stringify(meta)).catch(() => {});
|
|
260
|
+
await saveGeneratedVideoArtifact(ctx, filename, result.videoBuffer, meta);
|
|
197
261
|
invalidateHistoryIndex();
|
|
198
262
|
|
|
199
263
|
finishMeta = { filename, xaiVideoRequestId: result.xaiVideoRequestId };
|
|
@@ -206,7 +270,11 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
|
|
|
206
270
|
revisedPrompt: result.revisedPrompt,
|
|
207
271
|
elapsed,
|
|
208
272
|
usage: result.usage,
|
|
273
|
+
requestedModel: result.requestedModel,
|
|
274
|
+
effectiveModel: result.effectiveModel,
|
|
275
|
+
modelFallback: result.modelFallback,
|
|
209
276
|
video: meta.video,
|
|
277
|
+
videoContinuity,
|
|
210
278
|
...(meta.videoSeries ? { videoSeries: meta.videoSeries } : {}),
|
|
211
279
|
});
|
|
212
280
|
} catch (e) {
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { basename, join } from "node:path";
|
|
2
|
+
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
3
|
+
import { randomBytes } from "node:crypto";
|
|
4
|
+
import { requireRuntimeContext } from "../lib/runtimeContext.js";
|
|
5
|
+
import { getGrokProxyUrl } from "../lib/grokRuntime.js";
|
|
6
|
+
import { logEvent, logError } from "../lib/logger.js";
|
|
7
|
+
import { downloadVideo, pollVideoUntilDone } from "../lib/grokVideoAdapter.js";
|
|
8
|
+
import { invalidateHistoryIndex } from "../lib/historyIndex.js";
|
|
9
|
+
import { ACTIVE_VIDEO_PROMPT_GUIDANCE, appendVideoContinuityEntry, lineageFromVideoMetadata, readVideoSidecar } from "../lib/videoContinuity.js";
|
|
10
|
+
import { assertLocalMp4, extractVideoFrame, safeGeneratedFilePath } from "../lib/videoFrameExtract.js";
|
|
11
|
+
function videoProxyUrl(ctx, path) {
|
|
12
|
+
return { url: getGrokProxyUrl(ctx, path), headers: { "Content-Type": "application/json", Authorization: "Bearer dummy" } };
|
|
13
|
+
}
|
|
14
|
+
function routeError(message, status = 400) {
|
|
15
|
+
return Object.assign(new Error(message), { status });
|
|
16
|
+
}
|
|
17
|
+
function sendError(res, err) {
|
|
18
|
+
res.status(typeof err?.status === "number" ? err.status : 500).json({ error: err?.message || String(err) });
|
|
19
|
+
}
|
|
20
|
+
async function safeGeneratedFile(ctx, file, options = {}) {
|
|
21
|
+
return safeGeneratedFilePath(ctx.config.storage.generatedDir, file, options);
|
|
22
|
+
}
|
|
23
|
+
function summarizeSource(input) {
|
|
24
|
+
if (input.startsWith("data:video/")) {
|
|
25
|
+
const encoded = input.split(",", 2)[1] || "";
|
|
26
|
+
return { kind: "data-url", approximateBytes: Math.floor(encoded.length * 0.75) };
|
|
27
|
+
}
|
|
28
|
+
if (/^https?:\/\//i.test(input)) {
|
|
29
|
+
try {
|
|
30
|
+
const parsed = new URL(input);
|
|
31
|
+
return { kind: "url", origin: parsed.origin, pathname: basename(parsed.pathname) };
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return { kind: "url" };
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
if (/^file[-_][A-Za-z0-9._-]+$/.test(input))
|
|
38
|
+
return { kind: "file_id" };
|
|
39
|
+
return { kind: "generated-file", filename: basename(input) };
|
|
40
|
+
}
|
|
41
|
+
async function resolveVideoInput(ctx, input) {
|
|
42
|
+
if (/^https?:\/\//i.test(input) || input.startsWith("data:video/"))
|
|
43
|
+
return { url: input };
|
|
44
|
+
if (/^file[-_][A-Za-z0-9._-]+$/.test(input))
|
|
45
|
+
return { file_id: input };
|
|
46
|
+
const inputPath = await safeGeneratedFile(ctx, input, { requireMp4: true });
|
|
47
|
+
await assertLocalMp4(inputPath);
|
|
48
|
+
const buf = await readFile(inputPath);
|
|
49
|
+
return { url: `data:video/mp4;base64,${buf.toString("base64")}` };
|
|
50
|
+
}
|
|
51
|
+
function validateEditModel(model) {
|
|
52
|
+
if (typeof model !== "string")
|
|
53
|
+
throw routeError("model must be a string", 400);
|
|
54
|
+
if (model !== "grok-imagine-video")
|
|
55
|
+
throw routeError("Video edit/extension only supports grok-imagine-video", 400);
|
|
56
|
+
return model;
|
|
57
|
+
}
|
|
58
|
+
async function saveVideoResult(ctx, options) {
|
|
59
|
+
const { buffer, contentType } = await downloadVideo(ctx, options.videoUrl, options.signal);
|
|
60
|
+
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
61
|
+
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
62
|
+
const filename = `${Date.now()}_${rand}.mp4`;
|
|
63
|
+
const filePath = join(ctx.config.storage.generatedDir, filename);
|
|
64
|
+
const sourceFilename = /^https?:\/\//i.test(options.source) || options.source.startsWith("data:") || /^file[-_]/.test(options.source)
|
|
65
|
+
? null
|
|
66
|
+
: basename(options.source);
|
|
67
|
+
const sourceMeta = sourceFilename ? await readVideoSidecar(ctx.config.storage.generatedDir, sourceFilename) : null;
|
|
68
|
+
const parentLineage = sourceFilename ? lineageFromVideoMetadata(sourceFilename, sourceMeta) : null;
|
|
69
|
+
const videoContinuity = appendVideoContinuityEntry(parentLineage, {
|
|
70
|
+
filename,
|
|
71
|
+
userPrompt: options.prompt,
|
|
72
|
+
revisedPrompt: options.prompt,
|
|
73
|
+
createdAt: Date.now(),
|
|
74
|
+
});
|
|
75
|
+
await writeFile(filePath, buffer);
|
|
76
|
+
try {
|
|
77
|
+
await writeFile(`${filePath}.json`, JSON.stringify({
|
|
78
|
+
kind: "video",
|
|
79
|
+
mediaType: "video",
|
|
80
|
+
requestId: options.requestId,
|
|
81
|
+
prompt: options.prompt,
|
|
82
|
+
userPrompt: options.prompt,
|
|
83
|
+
provider: "grok",
|
|
84
|
+
model: options.model,
|
|
85
|
+
createdAt: Date.now(),
|
|
86
|
+
usage: options.usage ?? null,
|
|
87
|
+
revisedPrompt: options.prompt,
|
|
88
|
+
videoContinuity,
|
|
89
|
+
video: {
|
|
90
|
+
operation: options.operation,
|
|
91
|
+
duration: options.duration,
|
|
92
|
+
source: summarizeSource(options.source),
|
|
93
|
+
sourceUrl: summarizeSource(options.videoUrl),
|
|
94
|
+
contentType,
|
|
95
|
+
},
|
|
96
|
+
}));
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
await unlink(filePath).catch(() => { });
|
|
100
|
+
throw err;
|
|
101
|
+
}
|
|
102
|
+
invalidateHistoryIndex();
|
|
103
|
+
return { filename, url: `/generated/${encodeURIComponent(filename)}`, sourceUrl: options.videoUrl };
|
|
104
|
+
}
|
|
105
|
+
function requestSignal(req, res) {
|
|
106
|
+
const ac = new AbortController();
|
|
107
|
+
const abort = () => {
|
|
108
|
+
if (!res.writableEnded)
|
|
109
|
+
ac.abort();
|
|
110
|
+
};
|
|
111
|
+
req.on("aborted", abort);
|
|
112
|
+
res.on("close", abort);
|
|
113
|
+
return ac.signal;
|
|
114
|
+
}
|
|
115
|
+
function requirePrompt(value) {
|
|
116
|
+
return typeof value === "string" && value.trim() ? value : null;
|
|
117
|
+
}
|
|
118
|
+
function extractOutputText(data) {
|
|
119
|
+
const output = Array.isArray(data.output) ? data.output : [];
|
|
120
|
+
const texts = [];
|
|
121
|
+
for (const item of output) {
|
|
122
|
+
const content = item?.content;
|
|
123
|
+
if (!Array.isArray(content))
|
|
124
|
+
continue;
|
|
125
|
+
for (const part of content) {
|
|
126
|
+
if (part?.type === "output_text" && typeof part.text === "string")
|
|
127
|
+
texts.push(part.text);
|
|
128
|
+
if (part?.type === "text" && typeof part.text === "string")
|
|
129
|
+
texts.push(part.text);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return texts.join("\n").trim();
|
|
133
|
+
}
|
|
134
|
+
export function registerVideoExtendedRoutes(app, ctxRaw) {
|
|
135
|
+
const ctx = requireRuntimeContext(ctxRaw);
|
|
136
|
+
// --- Video Edit (V2V) ---
|
|
137
|
+
app.post("/api/video/edit", async (req, res) => {
|
|
138
|
+
try {
|
|
139
|
+
const { prompt: rawPrompt, videoUrl, model = "grok-imagine-video" } = req.body ?? {};
|
|
140
|
+
const prompt = requirePrompt(rawPrompt);
|
|
141
|
+
if (!prompt)
|
|
142
|
+
return res.status(400).json({ error: "prompt required", code: "PROMPT_REQUIRED", guidance: ACTIVE_VIDEO_PROMPT_GUIDANCE });
|
|
143
|
+
if (!videoUrl || typeof videoUrl !== "string")
|
|
144
|
+
return res.status(400).json({ error: "videoUrl required" });
|
|
145
|
+
const validModel = validateEditModel(model);
|
|
146
|
+
const signal = requestSignal(req, res);
|
|
147
|
+
const { url, headers } = videoProxyUrl(ctx, "/v1/videos/edits");
|
|
148
|
+
const video = await resolveVideoInput(ctx, videoUrl);
|
|
149
|
+
const apiRes = await fetch(url, { method: "POST", headers, body: JSON.stringify({ model: validModel, prompt, video }), signal });
|
|
150
|
+
if (!apiRes.ok) {
|
|
151
|
+
const t = await apiRes.text();
|
|
152
|
+
return res.status(apiRes.status).json({ error: t });
|
|
153
|
+
}
|
|
154
|
+
const { request_id } = (await apiRes.json());
|
|
155
|
+
if (!request_id)
|
|
156
|
+
return res.status(502).json({ error: "No request_id in response" });
|
|
157
|
+
logEvent("video", "edit:start", { requestId: request_id, model: validModel });
|
|
158
|
+
const result = await pollVideoUntilDone(ctx, request_id, { signal });
|
|
159
|
+
if (result.respectModeration === false)
|
|
160
|
+
return res.status(502).json({ error: "Grok video blocked by moderation" });
|
|
161
|
+
if (!result.videoUrl)
|
|
162
|
+
return res.status(502).json({ error: "No video URL in response" });
|
|
163
|
+
const saved = await saveVideoResult(ctx, { requestId: request_id, prompt, model: validModel, operation: "edit", source: videoUrl, duration: result.duration ?? null, videoUrl: result.videoUrl, usage: result.usage, signal });
|
|
164
|
+
logEvent("video", "edit:done", { requestId: request_id });
|
|
165
|
+
res.json({ requestId: request_id, url: saved.url, filename: saved.filename, sourceUrl: saved.sourceUrl, duration: result.duration, model: validModel });
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
logError("video", "edit:error", err);
|
|
169
|
+
sendError(res, err);
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
// --- Video Extension ---
|
|
173
|
+
app.post("/api/video/extend", async (req, res) => {
|
|
174
|
+
try {
|
|
175
|
+
const { prompt: rawPrompt, videoUrl, duration = 6, model = "grok-imagine-video" } = req.body ?? {};
|
|
176
|
+
const prompt = requirePrompt(rawPrompt);
|
|
177
|
+
if (!prompt)
|
|
178
|
+
return res.status(400).json({ error: "prompt required", code: "PROMPT_REQUIRED", guidance: ACTIVE_VIDEO_PROMPT_GUIDANCE });
|
|
179
|
+
if (!videoUrl || typeof videoUrl !== "string")
|
|
180
|
+
return res.status(400).json({ error: "videoUrl required" });
|
|
181
|
+
const validModel = validateEditModel(model);
|
|
182
|
+
const dur = Number(duration);
|
|
183
|
+
if (!Number.isInteger(dur) || dur < 2 || dur > 10)
|
|
184
|
+
return res.status(400).json({ error: "duration must be an integer between 2 and 10" });
|
|
185
|
+
const signal = requestSignal(req, res);
|
|
186
|
+
const { url, headers } = videoProxyUrl(ctx, "/v1/videos/extensions");
|
|
187
|
+
const video = await resolveVideoInput(ctx, videoUrl);
|
|
188
|
+
const apiRes = await fetch(url, { method: "POST", headers, body: JSON.stringify({ model: validModel, prompt, duration: dur, video }), signal });
|
|
189
|
+
if (!apiRes.ok) {
|
|
190
|
+
const t = await apiRes.text();
|
|
191
|
+
return res.status(apiRes.status).json({ error: t });
|
|
192
|
+
}
|
|
193
|
+
const { request_id } = (await apiRes.json());
|
|
194
|
+
if (!request_id)
|
|
195
|
+
return res.status(502).json({ error: "No request_id in response" });
|
|
196
|
+
logEvent("video", "extend:start", { requestId: request_id, model: validModel, duration: dur });
|
|
197
|
+
const result = await pollVideoUntilDone(ctx, request_id, { signal });
|
|
198
|
+
if (result.respectModeration === false)
|
|
199
|
+
return res.status(502).json({ error: "Grok video blocked by moderation" });
|
|
200
|
+
if (!result.videoUrl)
|
|
201
|
+
return res.status(502).json({ error: "No video URL in response" });
|
|
202
|
+
const saved = await saveVideoResult(ctx, { requestId: request_id, prompt, model: validModel, operation: "extend", source: videoUrl, duration: result.duration ?? null, videoUrl: result.videoUrl, usage: result.usage, signal });
|
|
203
|
+
logEvent("video", "extend:done", { requestId: request_id, totalDuration: result.duration });
|
|
204
|
+
res.json({ requestId: request_id, url: saved.url, filename: saved.filename, sourceUrl: saved.sourceUrl, duration: result.duration, model: validModel });
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
logError("video", "extend:error", err);
|
|
208
|
+
sendError(res, err);
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
// --- Video Frame Extraction ---
|
|
212
|
+
app.get("/api/video/frame", async (req, res) => {
|
|
213
|
+
try {
|
|
214
|
+
const file = req.query.file;
|
|
215
|
+
const position = req.query.position || "last";
|
|
216
|
+
if (!file)
|
|
217
|
+
return res.status(400).json({ error: "file query param required" });
|
|
218
|
+
const inputPath = await safeGeneratedFile(ctx, file, { requireMp4: true });
|
|
219
|
+
await assertLocalMp4(inputPath);
|
|
220
|
+
const tmpOut = join(ctx.config.storage.generatedDir, `frame_tmp_${randomBytes(4).toString("hex")}.png`);
|
|
221
|
+
try {
|
|
222
|
+
await extractVideoFrame(inputPath, tmpOut, position);
|
|
223
|
+
const frame = await readFile(tmpOut);
|
|
224
|
+
res.type("png").send(frame);
|
|
225
|
+
}
|
|
226
|
+
catch (err) {
|
|
227
|
+
return res.status(500).json({ error: "ffmpeg failed" });
|
|
228
|
+
}
|
|
229
|
+
finally {
|
|
230
|
+
await unlink(tmpOut).catch(() => { });
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
logError("video", "frame:error", err);
|
|
235
|
+
sendError(res, err);
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
// --- Video Analysis (Grok 4.3 Vision) ---
|
|
239
|
+
app.post("/api/video/analyze", async (req, res) => {
|
|
240
|
+
try {
|
|
241
|
+
const { videoUrl } = req.body ?? {};
|
|
242
|
+
if (!videoUrl || typeof videoUrl !== "string")
|
|
243
|
+
return res.status(400).json({ error: "videoUrl required" });
|
|
244
|
+
if (/^https?:\/\//i.test(videoUrl) || videoUrl.startsWith("data:")) {
|
|
245
|
+
return res.status(400).json({ error: "videoUrl must be a generated .mp4 filename" });
|
|
246
|
+
}
|
|
247
|
+
const input = await safeGeneratedFile(ctx, videoUrl, { requireMp4: true });
|
|
248
|
+
await assertLocalMp4(input);
|
|
249
|
+
const firstFrame = join(ctx.config.storage.generatedDir, `analyze_first_${randomBytes(4).toString("hex")}.png`);
|
|
250
|
+
const lastFrame = join(ctx.config.storage.generatedDir, `analyze_last_${randomBytes(4).toString("hex")}.png`);
|
|
251
|
+
try {
|
|
252
|
+
await extractVideoFrame(input, firstFrame, "0");
|
|
253
|
+
await extractVideoFrame(input, lastFrame, "last");
|
|
254
|
+
const first = (await readFile(firstFrame)).toString("base64");
|
|
255
|
+
const last = (await readFile(lastFrame)).toString("base64");
|
|
256
|
+
const { url, headers } = videoProxyUrl(ctx, "/v1/responses");
|
|
257
|
+
const apiRes = await fetch(url, {
|
|
258
|
+
method: "POST",
|
|
259
|
+
headers,
|
|
260
|
+
body: JSON.stringify({
|
|
261
|
+
model: "grok-4.3",
|
|
262
|
+
input: [{
|
|
263
|
+
role: "user",
|
|
264
|
+
content: [
|
|
265
|
+
{ type: "input_image", image_url: `data:image/png;base64,${first}`, detail: "high" },
|
|
266
|
+
{ type: "input_image", image_url: `data:image/png;base64,${last}`, detail: "high" },
|
|
267
|
+
{ type: "input_text", text: "Analyze these first and last frames from a video for recreation. Infer likely motion between them. Include shot type, camera movement, lighting, color palette, subjects, motion direction/speed, mood, and audio/sound prompt suggestions. Be specific and cinematic." },
|
|
268
|
+
],
|
|
269
|
+
}],
|
|
270
|
+
}),
|
|
271
|
+
});
|
|
272
|
+
if (!apiRes.ok) {
|
|
273
|
+
const t = await apiRes.text();
|
|
274
|
+
return res.status(apiRes.status).json({ error: t });
|
|
275
|
+
}
|
|
276
|
+
const data = (await apiRes.json());
|
|
277
|
+
const text = extractOutputText(data);
|
|
278
|
+
if (!text)
|
|
279
|
+
return res.status(502).json({ error: "No analysis text in response" });
|
|
280
|
+
logEvent("video", "analyze:done", { videoUrl, chars: text.length });
|
|
281
|
+
res.json({ analysis: text, model: "grok-4.3", method: "first-last-frame" });
|
|
282
|
+
}
|
|
283
|
+
finally {
|
|
284
|
+
await unlink(firstFrame).catch(() => { });
|
|
285
|
+
await unlink(lastFrame).catch(() => { });
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
catch (err) {
|
|
289
|
+
logError("video", "analyze:error", err);
|
|
290
|
+
sendError(res, err);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
}
|