ima2-gen 1.1.20 → 1.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +15 -25
  2. package/bin/commands/capabilities.js +2 -2
  3. package/bin/commands/capabilities.ts +2 -2
  4. package/bin/commands/defaults.js +2 -2
  5. package/bin/commands/defaults.ts +2 -2
  6. package/bin/commands/doctor.js +3 -3
  7. package/bin/commands/doctor.ts +3 -3
  8. package/bin/commands/edit.js +1 -1
  9. package/bin/commands/edit.ts +1 -1
  10. package/bin/commands/gen.js +1 -1
  11. package/bin/commands/gen.ts +1 -1
  12. package/bin/commands/grok.js +16 -11
  13. package/bin/commands/grok.ts +16 -11
  14. package/bin/commands/multimode.js +1 -1
  15. package/bin/commands/multimode.ts +1 -1
  16. package/bin/commands/observability.js +2 -2
  17. package/bin/commands/observability.ts +2 -2
  18. package/bin/commands/video.js +335 -13
  19. package/bin/commands/video.ts +249 -12
  20. package/bin/ima2.js +9 -9
  21. package/bin/ima2.ts +9 -9
  22. package/bin/lib/error-hints.js +2 -2
  23. package/bin/lib/error-hints.ts +2 -2
  24. package/docs/API.md +112 -3
  25. package/docs/CLI.md +61 -7
  26. package/docs/FAQ.ko.md +15 -20
  27. package/docs/FAQ.md +14 -19
  28. package/docs/NPX_QUICKSTART.md +40 -0
  29. package/docs/PROMPT_STUDIO.ko.md +1 -1
  30. package/docs/PROMPT_STUDIO.md +1 -1
  31. package/docs/README.ja.md +6 -16
  32. package/docs/README.ko.md +10 -20
  33. package/docs/README.zh-CN.md +7 -17
  34. package/docs/migration/runtime-test-inventory.md +8 -1
  35. package/lib/agentRuntime.js +19 -5
  36. package/lib/agentRuntime.ts +17 -5
  37. package/lib/capabilities.js +1 -1
  38. package/lib/capabilities.ts +1 -1
  39. package/lib/generationErrors.js +1 -1
  40. package/lib/generationErrors.ts +1 -1
  41. package/lib/grokProxyLauncher.js +26 -3
  42. package/lib/grokProxyLauncher.ts +27 -3
  43. package/lib/grokVideoAdapter.js +18 -89
  44. package/lib/grokVideoAdapter.ts +27 -88
  45. package/lib/grokVideoCanvas.js +25 -0
  46. package/lib/grokVideoCanvas.ts +26 -0
  47. package/lib/grokVideoDownload.js +58 -0
  48. package/lib/grokVideoDownload.ts +59 -0
  49. package/lib/grokVideoPlannerPrompt.js +64 -0
  50. package/lib/grokVideoPlannerPrompt.ts +67 -0
  51. package/lib/historyList.js +7 -1
  52. package/lib/historyList.ts +5 -1
  53. package/lib/oauthLauncher.js +21 -6
  54. package/lib/oauthLauncher.ts +22 -6
  55. package/lib/videoContinuity.js +149 -0
  56. package/lib/videoContinuity.ts +180 -0
  57. package/lib/videoFrameExtract.js +80 -0
  58. package/lib/videoFrameExtract.ts +78 -0
  59. package/node_modules/progrok/dist/index.js +187 -88
  60. package/node_modules/progrok/dist/index.js.map +1 -1
  61. package/node_modules/progrok/package.json +1 -1
  62. package/node_modules/progrok/skills/progrok/SKILL.md +33 -4
  63. package/package.json +2 -2
  64. package/routes/index.js +4 -0
  65. package/routes/index.ts +4 -0
  66. package/routes/quota.js +66 -0
  67. package/routes/quota.ts +89 -0
  68. package/routes/video.js +77 -15
  69. package/routes/video.ts +82 -14
  70. package/routes/videoExtended.js +293 -0
  71. package/routes/videoExtended.ts +284 -0
  72. package/server.js +6 -2
  73. package/server.ts +5 -2
  74. package/skills/ima2/SKILL.md +320 -7
  75. package/ui/dist/.vite/manifest.json +12 -12
  76. package/ui/dist/assets/{AgentWorkspace-DS8uvoLI.js → AgentWorkspace-B_hq9CLg.js} +2 -2
  77. package/ui/dist/assets/{CardNewsWorkspace-CYxMsE67.js → CardNewsWorkspace-wD12J7qk.js} +1 -1
  78. package/ui/dist/assets/{NodeCanvas-DccIc347.js → NodeCanvas-CI_wuPMf.js} +1 -1
  79. package/ui/dist/assets/{PromptBuilderPanel-BvxxwSJp.js → PromptBuilderPanel-CUTujJUV.js} +1 -1
  80. package/ui/dist/assets/{PromptImportDialog-u1_BFDRd.js → PromptImportDialog-CUi66jPK.js} +2 -2
  81. package/ui/dist/assets/{PromptImportDiscoverySection-C5uvkVSz.js → PromptImportDiscoverySection-Cm3vrjY4.js} +1 -1
  82. package/ui/dist/assets/{PromptImportFolderSection-D3E_O1SD.js → PromptImportFolderSection-DOtWTD9n.js} +1 -1
  83. package/ui/dist/assets/{PromptLibraryPanel-4gyf9CB9.js → PromptLibraryPanel-BMjQegRa.js} +2 -2
  84. package/ui/dist/assets/SettingsWorkspace-PiaVnsdA.js +1 -0
  85. package/ui/dist/assets/{index-DoKtXbod.js → index-31uVIdt4.js} +1 -1
  86. package/ui/dist/assets/index-CjgnNtgt.css +1 -0
  87. package/ui/dist/assets/index-Da2s4_-5.js +36 -0
  88. package/ui/dist/index.html +2 -2
  89. package/vendor/progrok-0.2.0.tgz +0 -0
  90. package/ui/dist/assets/SettingsWorkspace-F3eNu3mJ.js +0 -1
  91. package/ui/dist/assets/index-B6tcw_UF.css +0 -1
  92. package/ui/dist/assets/index-DYOh6gQD.js +0 -32
  93. package/vendor/progrok-0.1.1.tgz +0 -0
package/routes/video.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { mkdir, readFile, writeFile } from "fs/promises";
1
+ import { mkdir, readFile, unlink, writeFile } from "fs/promises";
2
2
  import { join } from "path";
3
3
  import { randomBytes } from "crypto";
4
4
  import type { Express, Request, Response } from "express";
@@ -8,6 +8,17 @@ import { logEvent, logError } from "../lib/logger.js";
8
8
  import { invalidateHistoryIndex } from "../lib/historyIndex.js";
9
9
  import { generateVideoViaGrok, type GrokVideoEvent } from "../lib/grokVideoAdapter.js";
10
10
  import { getVideoSeriesChain } from "../lib/videoSeriesChain.js";
11
+ import {
12
+ ACTIVE_VIDEO_PROMPT_GUIDANCE,
13
+ appendVideoContinuityEntry,
14
+ lineageFromVideoMetadata,
15
+ normalizeVideoContinuityLineage,
16
+ readVideoSidecar,
17
+ requireActiveVideoPrompt,
18
+ safeGeneratedVideoFilename,
19
+ type VideoContinuityLineage,
20
+ } from "../lib/videoContinuity.js";
21
+ import { extractGeneratedVideoFrameB64 } from "../lib/videoFrameExtract.js";
11
22
  import {
12
23
  normalizeGrokVideoModel,
13
24
  normalizeVideoResolution,
@@ -36,6 +47,17 @@ function isNormalizeError(x: unknown): x is NormalizeError {
36
47
  return typeof x === "object" && x !== null && typeof (x as { error?: unknown }).error === "string";
37
48
  }
38
49
 
50
+ export async function saveGeneratedVideoArtifact(ctx: RuntimeContext, filename: string, buffer: Buffer, metadata: unknown): Promise<void> {
51
+ const filePath = join(ctx.config.storage.generatedDir, filename);
52
+ await writeFile(filePath, buffer);
53
+ try {
54
+ await writeFile(`${filePath}.json`, JSON.stringify(metadata));
55
+ } catch (err) {
56
+ await unlink(filePath).catch(() => {});
57
+ throw err;
58
+ }
59
+ }
60
+
39
61
  async function resolveSourceImage(
40
62
  ctx: RuntimeContext,
41
63
  sourceImage: unknown,
@@ -44,6 +66,7 @@ async function resolveSourceImage(
44
66
  if (typeof sourceFilename === "string" && sourceFilename) {
45
67
  const safe = sourceFilename.replace(/^\/+/, "");
46
68
  if (safe.includes("..")) throw { status: 400, code: "GROK_VIDEO_INVALID_MODE", message: "invalid source filename" };
69
+ if (/\.mp4$/i.test(safe)) throw { status: 400, code: "GROK_VIDEO_INVALID_MODE", message: "use continueFromVideo for generated video continuation" };
47
70
  const buf = await readFile(join(ctx.config.storage.generatedDir, safe));
48
71
  return { b64: buf.toString("base64"), filename: safe };
49
72
  }
@@ -74,12 +97,12 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
74
97
  res.setHeader("Connection", "keep-alive");
75
98
  res.flushHeaders?.();
76
99
 
77
- const fail = (status: number | undefined, code: string, error: string) => {
100
+ const fail = (status: number | undefined, code: string, error: string, extra: Record<string, unknown> = {}) => {
78
101
  const httpStatus = status ?? 500;
79
102
  finishStatus = "error";
80
103
  finishHttpStatus = httpStatus;
81
104
  finishErrorCode = code;
82
- sendSse(res, "error", { error, code, status: httpStatus, requestId });
105
+ sendSse(res, "error", { error, code, status: httpStatus, requestId, ...extra });
83
106
  };
84
107
 
85
108
  try {
@@ -89,7 +112,8 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
89
112
  const topic = typeof req.body?.topic === "string" ? req.body.topic.trim() : "";
90
113
 
91
114
  if (provider !== "grok") return fail(400, "VIDEO_PROVIDER_UNSUPPORTED", "video generation requires provider 'grok'");
92
- if (typeof prompt !== "string" || !prompt.trim()) return fail(400, "PROMPT_REQUIRED", "Prompt is required");
115
+ const activePrompt = requireActiveVideoPrompt(prompt);
116
+ if (!activePrompt) return fail(400, "PROMPT_REQUIRED", "Prompt is required", { guidance: ACTIVE_VIDEO_PROMPT_GUIDANCE });
93
117
 
94
118
  const modelCheck = normalizeGrokVideoModel(rawModel);
95
119
  if (isNormalizeError(modelCheck)) return fail(modelCheck.status, modelCheck.code, modelCheck.error);
@@ -101,6 +125,20 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
101
125
  if (isNormalizeError(aspectCheck)) return fail(aspectCheck.status, aspectCheck.code, aspectCheck.error);
102
126
 
103
127
  // Resolve reference inputs: base64 list + existing-file list + legacy single source.
128
+ let parentLineage: VideoContinuityLineage | null = null;
129
+ let continueFromVideoFilename: string | null = null;
130
+ if (typeof req.body?.continueFromVideo === "string" && req.body.continueFromVideo.trim()) {
131
+ try {
132
+ continueFromVideoFilename = safeGeneratedVideoFilename(req.body.continueFromVideo);
133
+ const parentMeta = await readVideoSidecar(ctx.config.storage.generatedDir, continueFromVideoFilename);
134
+ parentLineage = lineageFromVideoMetadata(continueFromVideoFilename, parentMeta);
135
+ } catch (e: any) {
136
+ return fail(e?.status || 400, "GROK_VIDEO_INVALID_MODE", e?.message || "invalid continuation video");
137
+ }
138
+ } else {
139
+ parentLineage = normalizeVideoContinuityLineage(req.body?.continuityLineage);
140
+ }
141
+
104
142
  const refInputs: Array<{ image?: unknown; filename?: unknown }> = [
105
143
  ...toArray(req.body?.referenceImages).map((image) => ({ image })),
106
144
  ...toArray(req.body?.referenceFilenames).map((filename) => ({ filename })),
@@ -108,6 +146,13 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
108
146
  ? [{ image: req.body?.sourceImage, filename: req.body?.sourceFilename }]
109
147
  : []),
110
148
  ];
149
+ if (continueFromVideoFilename && !req.body?.sourceImage && !req.body?.sourceFilename) {
150
+ try {
151
+ refInputs.push({ image: await extractGeneratedVideoFrameB64(ctx.config.storage.generatedDir, continueFromVideoFilename) });
152
+ } catch (e: any) {
153
+ return fail(e?.status || 500, "GROK_VIDEO_FRAME_FAILED", e?.message || "failed to extract continuation frame");
154
+ }
155
+ }
111
156
  let resolved: Array<{ b64: string; filename: string | null }>;
112
157
  try {
113
158
  const all = await Promise.all(refInputs.map((r) => resolveSourceImage(ctx, r.image, r.filename)));
@@ -125,7 +170,7 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
125
170
  startJob({
126
171
  requestId,
127
172
  kind: "video",
128
- prompt,
173
+ prompt: activePrompt,
129
174
  meta: { kind: "video", sessionId, clientNodeId, model: modelCheck.model, mode, duration, resolution: resolutionCheck.resolution },
130
175
  });
131
176
  registerJobAbortController(requestId, cancelController);
@@ -137,7 +182,13 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
137
182
  const onEvent = (ev: GrokVideoEvent) => {
138
183
  if (ev.phase === "submitted") {
139
184
  setJobPhase(requestId, "streaming");
140
- sendSse(res, "submitted", { requestId, xaiVideoRequestId: ev.xaiVideoRequestId });
185
+ sendSse(res, "submitted", {
186
+ requestId,
187
+ xaiVideoRequestId: ev.xaiVideoRequestId,
188
+ requestedModel: ev.requestedModel,
189
+ effectiveModel: ev.effectiveModel,
190
+ modelFallback: ev.modelFallback ?? null,
191
+ });
141
192
  } else if (ev.phase === "progress") {
142
193
  sendSse(res, "progress", { requestId, progress: typeof ev.progress === "number" ? ev.progress / 100 : null, stalled: Boolean(ev.stalled) });
143
194
  } else {
@@ -147,10 +198,10 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
147
198
  };
148
199
 
149
200
  // Build prompt with series chain context
150
- const chain = topic ? await getVideoSeriesChain(ctx.config.storage.generatedDir, topic) : [];
201
+ const chain = !parentLineage && topic ? await getVideoSeriesChain(ctx.config.storage.generatedDir, topic) : [];
151
202
  const effectivePrompt = chain.length > 0
152
- ? `[Series topic: ${topic}]\n[Previous prompts in series:\n${chain.map((p, i) => `${i + 1}. ${p}`).join("\n")}\n]\n\n${prompt}`
153
- : prompt;
203
+ ? `[Series topic: ${topic}]\n[Previous prompts in series:\n${chain.map((p, i) => `${i + 1}. ${p}`).join("\n")}\n]\n\n${activePrompt}`
204
+ : activePrompt;
154
205
 
155
206
  const result = await generateVideoViaGrok(effectivePrompt, ctx, {
156
207
  model: modelCheck.model,
@@ -162,23 +213,33 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
162
213
  referenceImages,
163
214
  signal: cancelController.signal,
164
215
  requestId,
216
+ continuityLineage: parentLineage,
165
217
  onEvent,
166
218
  });
167
219
 
168
220
  const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
169
221
  const filename = `${Date.now()}_${rand}.mp4`;
170
222
  const elapsed = +((Date.now() - startTime) / 1000).toFixed(1);
223
+ const videoContinuity = appendVideoContinuityEntry(parentLineage, {
224
+ filename,
225
+ userPrompt: activePrompt,
226
+ revisedPrompt: result.revisedPrompt,
227
+ createdAt: Date.now(),
228
+ });
171
229
  const meta = {
172
230
  kind: "video",
173
231
  mediaType: "video",
174
232
  requestId,
175
233
  sessionId,
176
234
  clientNodeId,
177
- prompt,
178
- userPrompt: prompt,
235
+ prompt: activePrompt,
236
+ userPrompt: activePrompt,
179
237
  revisedPrompt: result.revisedPrompt,
180
238
  provider: "grok",
181
- model: modelCheck.model,
239
+ model: result.effectiveModel,
240
+ requestedModel: result.requestedModel,
241
+ effectiveModel: result.effectiveModel,
242
+ modelFallback: result.modelFallback,
182
243
  createdAt: Date.now(),
183
244
  elapsed,
184
245
  usage: result.usage,
@@ -189,11 +250,14 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
189
250
  aspectRatio: result.aspectRatio,
190
251
  sourceImageFilename: sourceFilename,
191
252
  xaiVideoRequestId: result.xaiVideoRequestId,
253
+ requestedModel: result.requestedModel,
254
+ effectiveModel: result.effectiveModel,
255
+ modelFallback: result.modelFallback,
192
256
  },
257
+ videoContinuity,
193
258
  ...(topic ? { videoSeries: { topic, chainIndex: chain.length } } : {}),
194
259
  };
195
- await writeFile(join(ctx.config.storage.generatedDir, filename), result.videoBuffer);
196
- await writeFile(join(ctx.config.storage.generatedDir, filename + ".json"), JSON.stringify(meta)).catch(() => {});
260
+ await saveGeneratedVideoArtifact(ctx, filename, result.videoBuffer, meta);
197
261
  invalidateHistoryIndex();
198
262
 
199
263
  finishMeta = { filename, xaiVideoRequestId: result.xaiVideoRequestId };
@@ -206,7 +270,11 @@ export function registerVideoRoutes(app: Express, ctxRaw: RouteRuntimeContext) {
206
270
  revisedPrompt: result.revisedPrompt,
207
271
  elapsed,
208
272
  usage: result.usage,
273
+ requestedModel: result.requestedModel,
274
+ effectiveModel: result.effectiveModel,
275
+ modelFallback: result.modelFallback,
209
276
  video: meta.video,
277
+ videoContinuity,
210
278
  ...(meta.videoSeries ? { videoSeries: meta.videoSeries } : {}),
211
279
  });
212
280
  } catch (e) {
@@ -0,0 +1,293 @@
1
+ import { basename, join } from "node:path";
2
+ import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
3
+ import { randomBytes } from "node:crypto";
4
+ import { requireRuntimeContext } from "../lib/runtimeContext.js";
5
+ import { getGrokProxyUrl } from "../lib/grokRuntime.js";
6
+ import { logEvent, logError } from "../lib/logger.js";
7
+ import { downloadVideo, pollVideoUntilDone } from "../lib/grokVideoAdapter.js";
8
+ import { invalidateHistoryIndex } from "../lib/historyIndex.js";
9
+ import { ACTIVE_VIDEO_PROMPT_GUIDANCE, appendVideoContinuityEntry, lineageFromVideoMetadata, readVideoSidecar } from "../lib/videoContinuity.js";
10
+ import { assertLocalMp4, extractVideoFrame, safeGeneratedFilePath } from "../lib/videoFrameExtract.js";
11
+ function videoProxyUrl(ctx, path) {
12
+ return { url: getGrokProxyUrl(ctx, path), headers: { "Content-Type": "application/json", Authorization: "Bearer dummy" } };
13
+ }
14
+ function routeError(message, status = 400) {
15
+ return Object.assign(new Error(message), { status });
16
+ }
17
+ function sendError(res, err) {
18
+ res.status(typeof err?.status === "number" ? err.status : 500).json({ error: err?.message || String(err) });
19
+ }
20
+ async function safeGeneratedFile(ctx, file, options = {}) {
21
+ return safeGeneratedFilePath(ctx.config.storage.generatedDir, file, options);
22
+ }
23
+ function summarizeSource(input) {
24
+ if (input.startsWith("data:video/")) {
25
+ const encoded = input.split(",", 2)[1] || "";
26
+ return { kind: "data-url", approximateBytes: Math.floor(encoded.length * 0.75) };
27
+ }
28
+ if (/^https?:\/\//i.test(input)) {
29
+ try {
30
+ const parsed = new URL(input);
31
+ return { kind: "url", origin: parsed.origin, pathname: basename(parsed.pathname) };
32
+ }
33
+ catch {
34
+ return { kind: "url" };
35
+ }
36
+ }
37
+ if (/^file[-_][A-Za-z0-9._-]+$/.test(input))
38
+ return { kind: "file_id" };
39
+ return { kind: "generated-file", filename: basename(input) };
40
+ }
41
+ async function resolveVideoInput(ctx, input) {
42
+ if (/^https?:\/\//i.test(input) || input.startsWith("data:video/"))
43
+ return { url: input };
44
+ if (/^file[-_][A-Za-z0-9._-]+$/.test(input))
45
+ return { file_id: input };
46
+ const inputPath = await safeGeneratedFile(ctx, input, { requireMp4: true });
47
+ await assertLocalMp4(inputPath);
48
+ const buf = await readFile(inputPath);
49
+ return { url: `data:video/mp4;base64,${buf.toString("base64")}` };
50
+ }
51
+ function validateEditModel(model) {
52
+ if (typeof model !== "string")
53
+ throw routeError("model must be a string", 400);
54
+ if (model !== "grok-imagine-video")
55
+ throw routeError("Video edit/extension only supports grok-imagine-video", 400);
56
+ return model;
57
+ }
58
+ async function saveVideoResult(ctx, options) {
59
+ const { buffer, contentType } = await downloadVideo(ctx, options.videoUrl, options.signal);
60
+ await mkdir(ctx.config.storage.generatedDir, { recursive: true });
61
+ const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
62
+ const filename = `${Date.now()}_${rand}.mp4`;
63
+ const filePath = join(ctx.config.storage.generatedDir, filename);
64
+ const sourceFilename = /^https?:\/\//i.test(options.source) || options.source.startsWith("data:") || /^file[-_]/.test(options.source)
65
+ ? null
66
+ : basename(options.source);
67
+ const sourceMeta = sourceFilename ? await readVideoSidecar(ctx.config.storage.generatedDir, sourceFilename) : null;
68
+ const parentLineage = sourceFilename ? lineageFromVideoMetadata(sourceFilename, sourceMeta) : null;
69
+ const videoContinuity = appendVideoContinuityEntry(parentLineage, {
70
+ filename,
71
+ userPrompt: options.prompt,
72
+ revisedPrompt: options.prompt,
73
+ createdAt: Date.now(),
74
+ });
75
+ await writeFile(filePath, buffer);
76
+ try {
77
+ await writeFile(`${filePath}.json`, JSON.stringify({
78
+ kind: "video",
79
+ mediaType: "video",
80
+ requestId: options.requestId,
81
+ prompt: options.prompt,
82
+ userPrompt: options.prompt,
83
+ provider: "grok",
84
+ model: options.model,
85
+ createdAt: Date.now(),
86
+ usage: options.usage ?? null,
87
+ revisedPrompt: options.prompt,
88
+ videoContinuity,
89
+ video: {
90
+ operation: options.operation,
91
+ duration: options.duration,
92
+ source: summarizeSource(options.source),
93
+ sourceUrl: summarizeSource(options.videoUrl),
94
+ contentType,
95
+ },
96
+ }));
97
+ }
98
+ catch (err) {
99
+ await unlink(filePath).catch(() => { });
100
+ throw err;
101
+ }
102
+ invalidateHistoryIndex();
103
+ return { filename, url: `/generated/${encodeURIComponent(filename)}`, sourceUrl: options.videoUrl };
104
+ }
105
+ function requestSignal(req, res) {
106
+ const ac = new AbortController();
107
+ const abort = () => {
108
+ if (!res.writableEnded)
109
+ ac.abort();
110
+ };
111
+ req.on("aborted", abort);
112
+ res.on("close", abort);
113
+ return ac.signal;
114
+ }
115
+ function requirePrompt(value) {
116
+ return typeof value === "string" && value.trim() ? value : null;
117
+ }
118
+ function extractOutputText(data) {
119
+ const output = Array.isArray(data.output) ? data.output : [];
120
+ const texts = [];
121
+ for (const item of output) {
122
+ const content = item?.content;
123
+ if (!Array.isArray(content))
124
+ continue;
125
+ for (const part of content) {
126
+ if (part?.type === "output_text" && typeof part.text === "string")
127
+ texts.push(part.text);
128
+ if (part?.type === "text" && typeof part.text === "string")
129
+ texts.push(part.text);
130
+ }
131
+ }
132
+ return texts.join("\n").trim();
133
+ }
134
+ export function registerVideoExtendedRoutes(app, ctxRaw) {
135
+ const ctx = requireRuntimeContext(ctxRaw);
136
+ // --- Video Edit (V2V) ---
137
+ app.post("/api/video/edit", async (req, res) => {
138
+ try {
139
+ const { prompt: rawPrompt, videoUrl, model = "grok-imagine-video" } = req.body ?? {};
140
+ const prompt = requirePrompt(rawPrompt);
141
+ if (!prompt)
142
+ return res.status(400).json({ error: "prompt required", code: "PROMPT_REQUIRED", guidance: ACTIVE_VIDEO_PROMPT_GUIDANCE });
143
+ if (!videoUrl || typeof videoUrl !== "string")
144
+ return res.status(400).json({ error: "videoUrl required" });
145
+ const validModel = validateEditModel(model);
146
+ const signal = requestSignal(req, res);
147
+ const { url, headers } = videoProxyUrl(ctx, "/v1/videos/edits");
148
+ const video = await resolveVideoInput(ctx, videoUrl);
149
+ const apiRes = await fetch(url, { method: "POST", headers, body: JSON.stringify({ model: validModel, prompt, video }), signal });
150
+ if (!apiRes.ok) {
151
+ const t = await apiRes.text();
152
+ return res.status(apiRes.status).json({ error: t });
153
+ }
154
+ const { request_id } = (await apiRes.json());
155
+ if (!request_id)
156
+ return res.status(502).json({ error: "No request_id in response" });
157
+ logEvent("video", "edit:start", { requestId: request_id, model: validModel });
158
+ const result = await pollVideoUntilDone(ctx, request_id, { signal });
159
+ if (result.respectModeration === false)
160
+ return res.status(502).json({ error: "Grok video blocked by moderation" });
161
+ if (!result.videoUrl)
162
+ return res.status(502).json({ error: "No video URL in response" });
163
+ const saved = await saveVideoResult(ctx, { requestId: request_id, prompt, model: validModel, operation: "edit", source: videoUrl, duration: result.duration ?? null, videoUrl: result.videoUrl, usage: result.usage, signal });
164
+ logEvent("video", "edit:done", { requestId: request_id });
165
+ res.json({ requestId: request_id, url: saved.url, filename: saved.filename, sourceUrl: saved.sourceUrl, duration: result.duration, model: validModel });
166
+ }
167
+ catch (err) {
168
+ logError("video", "edit:error", err);
169
+ sendError(res, err);
170
+ }
171
+ });
172
+ // --- Video Extension ---
173
+ app.post("/api/video/extend", async (req, res) => {
174
+ try {
175
+ const { prompt: rawPrompt, videoUrl, duration = 6, model = "grok-imagine-video" } = req.body ?? {};
176
+ const prompt = requirePrompt(rawPrompt);
177
+ if (!prompt)
178
+ return res.status(400).json({ error: "prompt required", code: "PROMPT_REQUIRED", guidance: ACTIVE_VIDEO_PROMPT_GUIDANCE });
179
+ if (!videoUrl || typeof videoUrl !== "string")
180
+ return res.status(400).json({ error: "videoUrl required" });
181
+ const validModel = validateEditModel(model);
182
+ const dur = Number(duration);
183
+ if (!Number.isInteger(dur) || dur < 2 || dur > 10)
184
+ return res.status(400).json({ error: "duration must be an integer between 2 and 10" });
185
+ const signal = requestSignal(req, res);
186
+ const { url, headers } = videoProxyUrl(ctx, "/v1/videos/extensions");
187
+ const video = await resolveVideoInput(ctx, videoUrl);
188
+ const apiRes = await fetch(url, { method: "POST", headers, body: JSON.stringify({ model: validModel, prompt, duration: dur, video }), signal });
189
+ if (!apiRes.ok) {
190
+ const t = await apiRes.text();
191
+ return res.status(apiRes.status).json({ error: t });
192
+ }
193
+ const { request_id } = (await apiRes.json());
194
+ if (!request_id)
195
+ return res.status(502).json({ error: "No request_id in response" });
196
+ logEvent("video", "extend:start", { requestId: request_id, model: validModel, duration: dur });
197
+ const result = await pollVideoUntilDone(ctx, request_id, { signal });
198
+ if (result.respectModeration === false)
199
+ return res.status(502).json({ error: "Grok video blocked by moderation" });
200
+ if (!result.videoUrl)
201
+ return res.status(502).json({ error: "No video URL in response" });
202
+ const saved = await saveVideoResult(ctx, { requestId: request_id, prompt, model: validModel, operation: "extend", source: videoUrl, duration: result.duration ?? null, videoUrl: result.videoUrl, usage: result.usage, signal });
203
+ logEvent("video", "extend:done", { requestId: request_id, totalDuration: result.duration });
204
+ res.json({ requestId: request_id, url: saved.url, filename: saved.filename, sourceUrl: saved.sourceUrl, duration: result.duration, model: validModel });
205
+ }
206
+ catch (err) {
207
+ logError("video", "extend:error", err);
208
+ sendError(res, err);
209
+ }
210
+ });
211
+ // --- Video Frame Extraction ---
212
+ app.get("/api/video/frame", async (req, res) => {
213
+ try {
214
+ const file = req.query.file;
215
+ const position = req.query.position || "last";
216
+ if (!file)
217
+ return res.status(400).json({ error: "file query param required" });
218
+ const inputPath = await safeGeneratedFile(ctx, file, { requireMp4: true });
219
+ await assertLocalMp4(inputPath);
220
+ const tmpOut = join(ctx.config.storage.generatedDir, `frame_tmp_${randomBytes(4).toString("hex")}.png`);
221
+ try {
222
+ await extractVideoFrame(inputPath, tmpOut, position);
223
+ const frame = await readFile(tmpOut);
224
+ res.type("png").send(frame);
225
+ }
226
+ catch (err) {
227
+ return res.status(500).json({ error: "ffmpeg failed" });
228
+ }
229
+ finally {
230
+ await unlink(tmpOut).catch(() => { });
231
+ }
232
+ }
233
+ catch (err) {
234
+ logError("video", "frame:error", err);
235
+ sendError(res, err);
236
+ }
237
+ });
238
+ // --- Video Analysis (Grok 4.3 Vision) ---
239
+ app.post("/api/video/analyze", async (req, res) => {
240
+ try {
241
+ const { videoUrl } = req.body ?? {};
242
+ if (!videoUrl || typeof videoUrl !== "string")
243
+ return res.status(400).json({ error: "videoUrl required" });
244
+ if (/^https?:\/\//i.test(videoUrl) || videoUrl.startsWith("data:")) {
245
+ return res.status(400).json({ error: "videoUrl must be a generated .mp4 filename" });
246
+ }
247
+ const input = await safeGeneratedFile(ctx, videoUrl, { requireMp4: true });
248
+ await assertLocalMp4(input);
249
+ const firstFrame = join(ctx.config.storage.generatedDir, `analyze_first_${randomBytes(4).toString("hex")}.png`);
250
+ const lastFrame = join(ctx.config.storage.generatedDir, `analyze_last_${randomBytes(4).toString("hex")}.png`);
251
+ try {
252
+ await extractVideoFrame(input, firstFrame, "0");
253
+ await extractVideoFrame(input, lastFrame, "last");
254
+ const first = (await readFile(firstFrame)).toString("base64");
255
+ const last = (await readFile(lastFrame)).toString("base64");
256
+ const { url, headers } = videoProxyUrl(ctx, "/v1/responses");
257
+ const apiRes = await fetch(url, {
258
+ method: "POST",
259
+ headers,
260
+ body: JSON.stringify({
261
+ model: "grok-4.3",
262
+ input: [{
263
+ role: "user",
264
+ content: [
265
+ { type: "input_image", image_url: `data:image/png;base64,${first}`, detail: "high" },
266
+ { type: "input_image", image_url: `data:image/png;base64,${last}`, detail: "high" },
267
+ { type: "input_text", text: "Analyze these first and last frames from a video for recreation. Infer likely motion between them. Include shot type, camera movement, lighting, color palette, subjects, motion direction/speed, mood, and audio/sound prompt suggestions. Be specific and cinematic." },
268
+ ],
269
+ }],
270
+ }),
271
+ });
272
+ if (!apiRes.ok) {
273
+ const t = await apiRes.text();
274
+ return res.status(apiRes.status).json({ error: t });
275
+ }
276
+ const data = (await apiRes.json());
277
+ const text = extractOutputText(data);
278
+ if (!text)
279
+ return res.status(502).json({ error: "No analysis text in response" });
280
+ logEvent("video", "analyze:done", { videoUrl, chars: text.length });
281
+ res.json({ analysis: text, model: "grok-4.3", method: "first-last-frame" });
282
+ }
283
+ finally {
284
+ await unlink(firstFrame).catch(() => { });
285
+ await unlink(lastFrame).catch(() => { });
286
+ }
287
+ }
288
+ catch (err) {
289
+ logError("video", "analyze:error", err);
290
+ sendError(res, err);
291
+ }
292
+ });
293
+ }