ima2-gen 1.1.18 → 1.1.20

package/lib/agentRuntime.ts

@@ -9,6 +9,8 @@ import { detectImageMimeFromB64 } from "./refs.js";
 import { resolveProviderOptions } from "./providerOptions.js";
 import { generateViaResponses } from "./responsesImageAdapter.js";
 import { generateViaGrok, type GrokReferenceImage } from "./grokImageAdapter.js";
+import { generateVideoViaGrok } from "./grokVideoAdapter.js";
+import { parseVideoParams } from "./agentGenerationPlanner.js";
 import {
   appendAgentTurn,
   buildImageContextManifest,
@@ -95,7 +97,7 @@ export async function runAgentGenerationPlan(
   const webSearchEnabled = options.provider === "grok" ? true : options.webSearchEnabled ?? session.webSearchEnabled;
   const enabledTools: AgentToolName[] = webSearchEnabled
     ? [...AGENT_ALLOWED_TOOLS]
-    : ["ima2.get_image_context", "ima2.generate_image"];
+    : ["ima2.get_image_context", "ima2.generate_image", "ima2.generate_video"];
   assertAgentAllowedTools(enabledTools);
   if (behavior.appendUserTurn !== false) {
     appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
@@ -111,6 +113,13 @@ export async function runAgentGenerationPlan(
     });
     return { assistantTurn, imageIds: [], webFindingIds: [] };
   }
+  if (plan.mode === "video") {
+    return runAgentVideoGeneration(ctx, sessionId, prompt, {
+      ...options,
+      requestId: options.requestId ?? `agent_video_${ulid()}`,
+      skipUserTurn: true,
+    });
+  }
   const manifest = buildImageContextManifest(sessionId);
   const contextStartedAt = Date.now();
   appendAgentTurn({
@@ -397,6 +406,117 @@ async function persistAgentImage(
   });
 }
 
+export async function runAgentVideoGeneration(
+  ctx: RuntimeContext,
+  sessionId: string,
+  prompt: string,
+  options: AgentRunOptions & { skipUserTurn?: boolean } = {},
+) {
+  const session = getAgentSession(sessionId);
+  if (!session) throw notFound(sessionId);
+  if (!options.skipUserTurn) {
+    appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
+  }
+  const requestId = options.requestId ?? `agent_video_${ulid()}`;
+  const startedAt = Date.now();
+
+  // Auto I2V: if session has a last image, use it as source
+  let sourceImage: string | undefined;
+  let mode: "text-to-video" | "image-to-video" = "text-to-video";
+  if (session.lastImageId) {
+    const images = getAgentImages(sessionId);
+    const lastImage = images.find((img) => img.id === session.lastImageId);
+    if (lastImage?.filename && !lastImage.filename.endsWith(".mp4")) {
+      try {
+        const { loadAssetB64 } = await import("./nodeStore.js");
+        sourceImage = await loadAssetB64(ctx.rootDir, lastImage.filename, ctx.config.storage.generatedDir);
+        mode = "image-to-video";
+      } catch { /* fallback to T2V */ }
+    }
+  }
+
+  const videoParams = parseVideoParams(prompt);
+
+  const result = await generateVideoViaGrok(prompt, ctx, {
+    model: "grok-imagine-video",
+    mode,
+    sourceImage,
+    duration: videoParams.duration ?? 5,
+    resolution: videoParams.resolution ?? "480p",
+    aspectRatio: (videoParams.aspectRatio ?? "auto") as "auto" | "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | "3:2" | "2:3",
+    requestId,
+    signal: options.signal ?? undefined,
+  });
+  const video = await persistAgentVideo(ctx, sessionId, prompt, requestId, result);
+  const finishedAt = Date.now();
+  const toolCall: AgentToolCallSummary = {
+    id: `tc_video_${ulid()}`,
+    name: "ima2.generate_video",
+    status: "complete",
+    startedAt,
+    finishedAt,
+    durationMs: finishedAt - startedAt,
+    requestId,
+    inputSummary: prompt,
+    outputSummary: `Generated video ${video.filename}.`,
+    imageIds: [video.id],
+  };
+  appendAgentTurn({
+    sessionId,
+    role: "tool",
+    text: "ima2.generate_video",
+    imageIds: [video.id],
+    status: "complete",
+    raw: { toolCalls: [toolCall] },
+  });
+  const assistantTurn = appendAgentTurn({
+    sessionId,
+    role: "assistant",
+    text: `Generated 1 video artifact. ${result.revisedPrompt}`,
+    imageIds: [video.id],
+    status: "complete",
+  });
+  return { assistantTurn, imageIds: [video.id], webFindingIds: [] };
+}
+
+async function persistAgentVideo(
+  ctx: RuntimeContext,
+  sessionId: string,
+  prompt: string,
+  requestId: string,
+  result: { videoBuffer: Buffer; revisedPrompt: string; usage: Record<string, number> | null; webSearchCalls: number },
+) {
+  await mkdir(ctx.config.storage.generatedDir, { recursive: true });
+  const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
+  const filename = `${Date.now()}_${rand}_agent.mp4`;
+  const meta = {
+    kind: "agent",
+    mediaType: "video",
+    requestId,
+    sessionId,
+    prompt,
+    userPrompt: prompt,
+    revisedPrompt: result.revisedPrompt,
+    provider: "grok",
+    model: "grok-imagine-video",
+    createdAt: Date.now(),
+    usage: result.usage,
+    webSearchCalls: result.webSearchCalls,
+  };
+  await writeFile(join(ctx.config.storage.generatedDir, filename), result.videoBuffer);
+  await writeFile(join(ctx.config.storage.generatedDir, `${filename}.json`), JSON.stringify(meta)).catch(() => {});
+  invalidateHistoryIndex();
+  logEvent("agent", "video_saved", { requestId, sessionId, filename });
+  return importAgentImage(sessionId, {
+    id: `ai_${ulid()}`,
+    filename,
+    url: `/generated/${filename}`,
+    prompt,
+    revisedPrompt: result.revisedPrompt,
+    createdAt: Date.now(),
+  });
+}
+
 function recordSearchFindings(sessionId: string, prompt: string, count: number, provider: string) {
   if (!count) return [];
   const isGrok = provider === "grok";

package/lib/agentTypes.js

@@ -2,4 +2,5 @@ export const AGENT_ALLOWED_TOOLS = [
     "ima2.get_image_context",
     "ima2.web_search",
     "ima2.generate_image",
+    "ima2.generate_video",
 ];

package/lib/agentTypes.ts

@@ -2,6 +2,7 @@ export const AGENT_ALLOWED_TOOLS = [
   "ima2.get_image_context",
   "ima2.web_search",
   "ima2.generate_image",
+  "ima2.generate_video",
 ] as const;
 
 export type AgentToolName = typeof AGENT_ALLOWED_TOOLS[number];
@@ -11,7 +12,7 @@ export type AgentToolCallStatus = "queued" | "running" | "complete" | "error";
 export type AgentQueueStatus = "queued" | "running" | "succeeded" | "failed" | "canceled";
 export type AgentSessionRunStatus = "idle" | "queued" | "running" | "error";
 export type AgentGenerationStrategy = "auto" | "manual";
-export type AgentGenerationPlanMode = "single" | "fanout" | "question";
+export type AgentGenerationPlanMode = "single" | "fanout" | "question" | "video";
 export type AgentGenerationPlanSource = "auto-default" | "auto-request" | "manual-settings" | "slash-command" | "question-command";
 export type AgentSlashCommandName = "question" | "help" | "variants" | "generate" | "parallelism";
 

package/lib/assetLifecycle.js

@@ -1,5 +1,5 @@
 import { getDb } from "./db.js";
-import { rename, unlink, access } from "fs/promises";
+import { mkdir, rename, unlink, access } from "fs/promises";
 import { resolve, sep } from "path";
 import { moveToSystemTrash } from "./systemTrash.js";
 import { config } from "../config.js";
@@ -87,21 +87,25 @@ export async function trashAsset(rootDir, filename) {
         paths.push(sidecar);
     }
     catch { }
+    let trashMethod = "system";
     try {
         await moveToSystemTrash(paths);
     }
-    catch (cause) {
-        const err = new Error("Could not move asset to system trash");
-        err.status = 500;
-        err.code = "SYSTEM_TRASH_FAILED";
-        err.cause = cause;
-        throw err;
+    catch {
+        trashMethod = "internal";
+        const trashDir = resolve(config.storage.trashDir);
+        await mkdir(trashDir, { recursive: true });
+        const trashId = `${Date.now()}_${filename}`;
+        for (const p of paths) {
+            const dest = resolve(trashDir, p.endsWith(".json") ? `${trashId}.json` : trashId);
+            await rename(p, dest);
+        }
     }
     const summary = markNodesAssetMissing(filename);
     return {
         ok: true,
         filename,
-        trash: "system",
+        trash: trashMethod,
         undoableInApp: false,
         sessionsTouched: summary.sessionsTouched,
         nodesTouched: summary.nodesTouched,

package/lib/assetLifecycle.ts

@@ -1,5 +1,5 @@
 import { getDb } from "./db.js";
-import { rename, unlink, access } from "fs/promises";
+import { mkdir, rename, unlink, access } from "fs/promises";
 import { resolve, sep } from "path";
 import { moveToSystemTrash } from "./systemTrash.js";
 import { config } from "../config.js";
@@ -84,21 +84,25 @@ export async function trashAsset(rootDir: string, filename: string) {
     paths.push(sidecar);
   } catch {}
 
+  let trashMethod: "system" | "internal" = "system";
   try {
     await moveToSystemTrash(paths);
-  } catch (cause) {
-    const err: any = new Error("Could not move asset to system trash");
-    err.status = 500;
-    err.code = "SYSTEM_TRASH_FAILED";
-    err.cause = cause;
-    throw err;
+  } catch {
+    trashMethod = "internal";
+    const trashDir = resolve(config.storage.trashDir);
+    await mkdir(trashDir, { recursive: true });
+    const trashId = `${Date.now()}_${filename}`;
+    for (const p of paths) {
+      const dest = resolve(trashDir, p.endsWith(".json") ? `${trashId}.json` : trashId);
+      await rename(p, dest);
+    }
   }
 
   const summary = markNodesAssetMissing(filename);
   return {
     ok: true,
     filename,
-    trash: "system",
+    trash: trashMethod,
     undoableInApp: false,
     sessionsTouched: summary.sessionsTouched,
     nodesTouched: summary.nodesTouched,

package/lib/capabilities.js

@@ -106,7 +106,7 @@ export function buildIma2Capabilities({ appConfig = runtimeConfigDefault, packag
             i2i: "Use --ref for reference generation, or ima2 edit <file> --prompt \"<text>\" for image edits.",
             defaults: "Use ima2 defaults set model/reasoning for persistent defaults; request flags remain per-call overrides.",
             promptBuilder: "Use ima2 prompt build --message \"...\" to refine prompt intent. Use ima2 gen / ima2 multimode to generate images. Workspace profile settings are UI-only.",
-            video: "Use ima2 video \"<prompt>\" to generate video. Supports --ref for image-to-video and reference-to-video modes.",
+            video: "Use ima2 video \"<prompt>\" to generate video. Supports --ref for image-to-video and reference-to-video modes. Use --topic for series continuity across multiple generations.",
         },
     };
 }

package/lib/capabilities.ts

@@ -120,7 +120,7 @@ export function buildIma2Capabilities({
       i2i: "Use --ref for reference generation, or ima2 edit <file> --prompt \"<text>\" for image edits.",
       defaults: "Use ima2 defaults set model/reasoning for persistent defaults; request flags remain per-call overrides.",
       promptBuilder: "Use ima2 prompt build --message \"...\" to refine prompt intent. Use ima2 gen / ima2 multimode to generate images. Workspace profile settings are UI-only.",
-      video: "Use ima2 video \"<prompt>\" to generate video. Supports --ref for image-to-video and reference-to-video modes.",
+      video: "Use ima2 video \"<prompt>\" to generate video. Supports --ref for image-to-video and reference-to-video modes. Use --topic for series continuity across multiple generations.",
     },
   };
 }

package/lib/grokVideoAdapter.js

@@ -45,6 +45,25 @@ function sourceImageUrl(image, mime) {
     const detected = mime || detectImageMimeFromB64(image) || "image/png";
     return `data:${detected};base64,${image}`;
 }
+/** Map aspect ratio + resolution to pixel dimensions for white canvas injection. */
+function aspectToCanvas(aspectRatio, resolution) {
+    const base = resolution === "720p" ? 720 : 480;
+    const ratios = {
+        "16:9": [16, 9], "9:16": [9, 16], "4:3": [4, 3], "3:4": [3, 4],
+        "3:2": [3, 2], "2:3": [2, 3], "1:1": [1, 1], "auto": [16, 9],
+    };
+    const [w, h] = ratios[aspectRatio] || [16, 9];
+    if (w >= h)
+        return { width: Math.round(base * w / h), height: base };
+    return { width: base, height: Math.round(base * h / w) };
+}
+/** Generate a minimal white PNG as base64 (no external deps). */
+function generateWhiteCanvasB64() {
+    // Minimal valid 1x1 white PNG, scaled conceptually — xAI will accept any valid PNG
+    // For simplicity, use a tiny white PNG (the model doesn't use it as a real frame)
+    const PNG_1x1_WHITE = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/58BAwAHBQKhPX8EPAAAAABJRU5ErkJggg==";
+    return PNG_1x1_WHITE;
+}
 const FAILED_CODE_MAP = {
     invalid_argument: { code: "GROK_VIDEO_REQUEST_FAILED", status: 400 },
     permission_denied: { code: "GROK_VIDEO_REQUEST_FAILED", status: 403 },
@@ -383,11 +402,20 @@ export async function generateVideoViaGrok(prompt, ctx, options = {}) {
     const payload = buildVideoGenerationPayload(plan, { model, sourceImageUrl: srcUrl, referenceImageUrls: refUrls });
     let xaiVideoRequestId;
     let effectiveModel = model;
+    // grokv1.5 doesn't support T2V — inject a white canvas as source image to use I2V path
+    let effectivePayload = payload;
+    if (model === "grok-imagine-video-1.5-preview" && !srcUrl && refUrls.length === 0) {
+        const { width, height } = aspectToCanvas(plan.aspectRatio, plan.resolution);
+        const whiteCanvas = generateWhiteCanvasB64();
+        const canvasSrcUrl = `data:image/png;base64,${whiteCanvas}`;
+        effectivePayload = buildVideoGenerationPayload({ ...plan, prompt: `${plan.prompt}. This is not a start frame — generate freely as a new video.` }, { model, sourceImageUrl: canvasSrcUrl, referenceImageUrls: [] });
+        logEvent("grok", "video:1.5-t2v-canvas", { requestId: options.requestId, width, height });
+    }
     try {
-        xaiVideoRequestId = await startVideoRequest(ctx, payload, options);
+        xaiVideoRequestId = await startVideoRequest(ctx, effectivePayload, options);
     }
     catch (e) {
-        // Fallback: if 1.5-preview fails, retry with base model
+        // Fallback: if 1.5-preview still fails, retry with base model
         if (model !== "grok-imagine-video" && e?.status === 400) {
             effectiveModel = "grok-imagine-video";
             const fallbackPayload = buildVideoGenerationPayload(plan, { model: effectiveModel, sourceImageUrl: srcUrl, referenceImageUrls: refUrls });

package/lib/grokVideoAdapter.ts

@@ -124,6 +124,26 @@ function sourceImageUrl(image: string, mime?: string | null): string {
   return `data:${detected};base64,${image}`;
 }
 
+/** Map aspect ratio + resolution to pixel dimensions for white canvas injection. */
+function aspectToCanvas(aspectRatio: string, resolution: string): { width: number; height: number } {
+  const base = resolution === "720p" ? 720 : 480;
+  const ratios: Record<string, [number, number]> = {
+    "16:9": [16, 9], "9:16": [9, 16], "4:3": [4, 3], "3:4": [3, 4],
+    "3:2": [3, 2], "2:3": [2, 3], "1:1": [1, 1], "auto": [16, 9],
+  };
+  const [w, h] = ratios[aspectRatio] || [16, 9];
+  if (w >= h) return { width: Math.round(base * w / h), height: base };
+  return { width: base, height: Math.round(base * h / w) };
+}
+
+/** Generate a minimal white PNG as base64 (no external deps). */
+function generateWhiteCanvasB64(): string {
+  // Minimal valid 1x1 white PNG, scaled conceptually — xAI will accept any valid PNG
+  // For simplicity, use a tiny white PNG (the model doesn't use it as a real frame)
+  const PNG_1x1_WHITE = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/58BAwAHBQKhPX8EPAAAAABJRU5ErkJggg==";
+  return PNG_1x1_WHITE;
+}
+
 const FAILED_CODE_MAP: Record<string, { code: string; status: number }> = {
   invalid_argument: { code: "GROK_VIDEO_REQUEST_FAILED", status: 400 },
   permission_denied: { code: "GROK_VIDEO_REQUEST_FAILED", status: 403 },
@@ -449,10 +469,24 @@ export async function generateVideoViaGrok(prompt: string, ctx: RouteRuntimeCont
   const payload = buildVideoGenerationPayload(plan, { model, sourceImageUrl: srcUrl, referenceImageUrls: refUrls });
   let xaiVideoRequestId: string;
   let effectiveModel = model;
+
+  // grokv1.5 doesn't support T2V — inject a white canvas as source image to use I2V path
+  let effectivePayload = payload;
+  if (model === "grok-imagine-video-1.5-preview" && !srcUrl && refUrls.length === 0) {
+    const { width, height } = aspectToCanvas(plan.aspectRatio, plan.resolution);
+    const whiteCanvas = generateWhiteCanvasB64();
+    const canvasSrcUrl = `data:image/png;base64,${whiteCanvas}`;
+    effectivePayload = buildVideoGenerationPayload(
+      { ...plan, prompt: `${plan.prompt}. This is not a start frame — generate freely as a new video.` },
+      { model, sourceImageUrl: canvasSrcUrl, referenceImageUrls: [] },
+    );
+    logEvent("grok", "video:1.5-t2v-canvas", { requestId: options.requestId, width, height });
+  }
+
   try {
-    xaiVideoRequestId = await startVideoRequest(ctx, payload, options);
+    xaiVideoRequestId = await startVideoRequest(ctx, effectivePayload, options);
   } catch (e: any) {
-    // Fallback: if 1.5-preview fails, retry with base model
+    // Fallback: if 1.5-preview still fails, retry with base model
     if (model !== "grok-imagine-video" && e?.status === 400) {
       effectiveModel = "grok-imagine-video";
       const fallbackPayload = buildVideoGenerationPayload(plan, { model: effectiveModel, sourceImageUrl: srcUrl, referenceImageUrls: refUrls });

package/lib/historyList.js

@@ -34,6 +34,7 @@ export async function listHistoryRows(baseDir = config.storage.generatedDir) {
             url: `/generated/${rel.split("/").map(encodeURIComponent).join("/")}`,
             mediaType: meta?.mediaType || (/\.mp4$/i.test(name) ? "video" : "image"),
             video: meta?.video || null,
+            videoSeries: meta?.videoSeries || null,
             createdAt: meta?.createdAt || st?.mtimeMs || 0,
             prompt: meta?.prompt || null,
             userPrompt: meta?.userPrompt || meta?.prompt || null,

package/lib/historyList.ts

@@ -36,6 +36,7 @@ export async function listHistoryRows(baseDir = config.storage.generatedDir) {
       url: `/generated/${rel.split("/").map(encodeURIComponent).join("/")}`,
       mediaType: meta?.mediaType || (/\.mp4$/i.test(name) ? "video" : "image"),
       video: meta?.video || null,
+      videoSeries: meta?.videoSeries || null,
       createdAt: meta?.createdAt || st?.mtimeMs || 0,
       prompt: meta?.prompt || null,
       userPrompt: meta?.userPrompt || meta?.prompt || null,

package/lib/videoSeriesChain.js

@@ -0,0 +1,24 @@
+import { readdir, readFile } from "fs/promises";
+import { join } from "path";
+/**
+ * Scan generatedDir for videos with matching topic, return the most recent N revisedPrompts.
+ */
+export async function getVideoSeriesChain(generatedDir, topic, limit = 4) {
+    if (!topic.trim())
+        return [];
+    const entries = await readdir(generatedDir).catch(() => []);
+    const sidecars = entries.filter((e) => e.endsWith(".mp4.json"));
+    const matches = [];
+    for (const sidecar of sidecars) {
+        try {
+            const raw = await readFile(join(generatedDir, sidecar), "utf-8");
+            const meta = JSON.parse(raw);
+            if (meta.videoSeries?.topic === topic && meta.revisedPrompt) {
+                matches.push({ revisedPrompt: meta.revisedPrompt, createdAt: meta.createdAt ?? 0 });
+            }
+        }
+        catch { /* skip unreadable */ }
+    }
+    matches.sort((a, b) => b.createdAt - a.createdAt);
+    return matches.slice(0, limit).reverse().map((m) => m.revisedPrompt);
+}

package/lib/videoSeriesChain.ts

@@ -0,0 +1,29 @@
+import { readdir, readFile } from "fs/promises";
+import { join } from "path";
+
+interface VideoSeriesMeta {
+  revisedPrompt?: string;
+  createdAt?: number;
+  videoSeries?: { topic: string; chainIndex?: number };
+}
+
+/**
+ * Scan generatedDir for videos with matching topic, return the most recent N revisedPrompts.
+ */
+export async function getVideoSeriesChain(generatedDir: string, topic: string, limit = 4): Promise<string[]> {
+  if (!topic.trim()) return [];
+  const entries = await readdir(generatedDir).catch(() => [] as string[]);
+  const sidecars = entries.filter((e) => e.endsWith(".mp4.json"));
+  const matches: Array<{ revisedPrompt: string; createdAt: number }> = [];
+  for (const sidecar of sidecars) {
+    try {
+      const raw = await readFile(join(generatedDir, sidecar), "utf-8");
+      const meta: VideoSeriesMeta = JSON.parse(raw);
+      if (meta.videoSeries?.topic === topic && meta.revisedPrompt) {
+        matches.push({ revisedPrompt: meta.revisedPrompt, createdAt: meta.createdAt ?? 0 });
+      }
+    } catch { /* skip unreadable */ }
+  }
+  matches.sort((a, b) => b.createdAt - a.createdAt);
+  return matches.slice(0, limit).reverse().map((m) => m.revisedPrompt);
+}