@effectnode/media 0.5.0 → 0.6.0

package/dist/backend/movie-backend/agent/agent-backend.d.ts

@@ -4,7 +4,7 @@ import { type Application } from "express";
  * Persists the result to studio/:projectId/data/*.json and returns the parsed
  * result. Used both by the HTTP route below and by the generation queue worker.
  */
-export declare function generateMovieStudioBible(projectId: string, idea: string, model?: string): Promise<{
+export declare function generateMovieStudioBible(projectId: string, idea: string, model?: string, onProgress?: (statusText: string, current: number, total: number) => void): Promise<{
     characters: any[];
     places: any[];
     scenes: any[];

package/dist/backend/movie-backend/agent/agent-backend.js

@@ -48,7 +48,7 @@ function toOpenAIMessages(messages) {
 function buildSystemPrompt() {
     const toolList = TOOLS.map((t) => `- ${t.name}: ${t.description}`).join("\n");
     return [
-        "You are a media assistant that helps the user edit images or render videos in their workspace.",
+        "You are a media assistant that helps the user render videos in their workspace.",
         "",
         "Available tools:",
         toolList,
@@ -56,34 +56,16 @@ function buildSystemPrompt() {
         "Rules:",
         "- Call a tool only when you actually need its result. Once you have enough information, answer the user directly WITHOUT calling any tool.",
         "- Do not call the same tool twice with the same arguments.",
-        "- ALWAYS confirm with the user before generating or editing. Never call `image_to_video_generation`, `text_to_video_generation`, or `edit_image` until the user has explicitly said yes.",
+        "- ALWAYS confirm with the user before generating. Never call `image_to_video_generation` or `text_to_video_generation` until the user has explicitly said yes.",
         "",
         "## Getting started",
         "",
         'When the user greets you (e.g. says "hi" or "hello") or asks for help, FIRST ask them what they want to do:',
-        "1. Edit an image",
-        "2. Render a video from text",
-        "3. Render a video from an image",
+        "1. Render a video from text",
+        "2. Render a video from an image",
         "",
         "Then follow the matching workflow below.",
         "",
-        "## Workflow: Edit an image",
-        "",
-        "### Step 1 — Image",
-        "- Call `list_files` to see what is in the agent workspace.",
-        '- List every image file (files whose `kind` is "image") to the user as a numbered list so they can pick one.',
-        "- You may call `show_image` to preview an image before the user decides.",
-        "- If there are no images, tell the user and ask them to upload one first.",
-        "",
-        "### Step 2 — Edit instruction",
-        "- Plan the edit together with the user. Ask what they want to change: background, style, color, lighting, expression, object, and so on.",
-        "- Confirm the wording of the edit prompt before moving on.",
-        "",
-        "### Step 3 — Summary and confirmation",
-        "- Prepare a short summary containing the image and the edit instruction, and ask the user to confirm.",
-        "- Do NOT start editing yet. Wait for the user to say yes.",
-        "- Only after the user confirms, call `edit_image` with the image and prompt.",
-        "",
         "## Workflow: Render a video",
         "",
         "To generate a video you need a motion PROMPT and a DURATION. An IMAGE is optional but recommended.",
@@ -151,53 +133,80 @@ function sanitizeTrace(messages) {
     });
 }
 // ========== Movie Studio ==========
-const MOVIE_STUDIO_CHARACTERS_PROMPT = [
+const MOVIE_STUDIO_CHARACTERS_PLAN_PROMPT = [
     "You are a movie pre-production planner. Given a movie or story idea, identify the characters.",
     "",
     "Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
     "",
-    '{"artStyle":"string","characters":[{"slug":"string","name":"string","imagePrompt":"string"}]}',
+    '{"artStyle":"string","characters":[{"slug":"string","name":"string"}]}',
     "",
     "Rules:",
     '- "slug" is a short lowercase hyphenated identifier (e.g. "the-lamb").',
     '- "artStyle" is ONE consistent art style for the entire film (e.g. cinematic photorealistic, 3D animation, hand-painted watercolor, anime).',
     '- If the idea does not mention an art style, set "artStyle" to "photo realistic render".',
-    "- A character's imagePrompt is a standalone text-to-image prompt that begins with the character's name and fully describes their appearance (age, face, build, outfit, distinctive features) together with their cultural background — the era/age, culture, ethnicity, and region the character belongs to — so the generated image is historically and culturally accurate and consistent.",
-    "- Apply the same artStyle to every character imagePrompt.",
-    "- Write prompts as natural-language English sentences, never comma-separated keyword tags.",
 ].join("\n");
-const MOVIE_STUDIO_PLACES_PROMPT = [
+const CHARACTER_IMAGE_PROMPT = [
+    "You are a movie pre-production planner. Given a movie idea, its art style, and one character, write that character's standalone image prompt.",
+    "",
+    "Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
+    "",
+    '{"imagePrompt":"string"}',
+    "",
+    "Rules:",
+    "- The imagePrompt begins with the character's name and fully describes their appearance (age, face, build, outfit, distinctive features) together with their cultural background — the era/age, culture, ethnicity, and region the character belongs to — so the generated image is historically and culturally accurate and consistent.",
+    "- Apply the given artStyle.",
+    "- Write the prompt as natural-language English sentences, never comma-separated keyword tags.",
+].join("\n");
+const MOVIE_STUDIO_PLACES_PLAN_PROMPT = [
     "You are a movie pre-production planner. Given a movie idea, its art style, and its characters, identify the places/locations.",
     "",
     "Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
     "",
-    '{"places":[{"slug":"string","name":"string","imagePrompt":"string"}]}',
+    '{"places":[{"slug":"string","name":"string"}]}',
     "",
     "Rules:",
     '- "slug" is a short lowercase hyphenated identifier (e.g. "sunny-meadow").',
-    "- A place's imagePrompt is a standalone text-to-image prompt that fully describes the location/environment (time of day, lighting, atmosphere, visual style).",
-    "- Apply the given artStyle to every place imagePrompt so it matches the characters.",
-    "- Write prompts as natural-language English sentences, never comma-separated keyword tags.",
 ].join("\n");
-const MOVIE_STUDIO_SCENES_PROMPT = [
+const PLACE_IMAGE_PROMPT = [
+    "You are a movie pre-production planner. Given a movie idea, its art style, and one place, write that place's standalone image prompt.",
+    "",
+    "Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
+    "",
+    '{"imagePrompt":"string"}',
+    "",
+    "Rules:",
+    "- The imagePrompt fully describes the location/environment (time of day, lighting, atmosphere, visual style).",
+    "- Apply the given artStyle.",
+    "- Write the prompt as natural-language English sentences, never comma-separated keyword tags.",
+].join("\n");
+const MOVIE_STUDIO_SCENES_PLAN_PROMPT = [
     "You are a movie pre-production planner. Given a movie idea, its art style, its characters, and its places, break the story into scenes.",
     "",
     "Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
     "",
-    '{"scenes":[{"slug":"string","duration":number,"description":"string","characterSlugs":["string"],"placeSlug":"string","scriptLines":[{"characterSlug":"string","line":"string"}],"voiceOver":"string","imagePrompt":"string"}]}',
+    '{"scenes":[{"slug":"string","duration":number,"characterSlugs":["string"],"placeSlug":"string"}]}',
     "",
     "Rules:",
-    "- Break the story into many small scenes: each scene is a single camera moment with one action beat.",
-    "- Generate as many scenes as possible: maximize the scene count. Every distinct beat, camera change, or exchange of dialogue should become its own scene — when in doubt, split into more scenes rather than fewer.",
-    "- Never reduce or summarize the dialogue: distribute every single spoken line across the resulting small scenes so no line is lost.",
-    "- Preserve as much detail as possible in every description and imagePrompt.",
+    '- "slug" is a short lowercase hyphenated identifier (e.g. "opening-meadow").',
+    "- Break the story into a natural number of scenes. Group related action beats and dialogue together — do not inflate the scene count by splitting every moment into its own scene.",
+    "- Never reduce or summarize the dialogue: keep every spoken line, grouping multiple lines into a scene where it reads naturally.",
     "- Each scene references the characters and the place involved via their slugs (characterSlugs is a list; placeSlug is a single slug).",
-    "- A scene's imagePrompt is ONE coherent shot that combines the referenced characters AND the place together, using the given artStyle.",
-    "- scriptLines is one entry per spoken line, each with the speaking character's slug and the exact spoken line. Include every single line of dialogue in the scene.",
-    "- voiceOver is the narration/voiceover for that scene (use an empty string when there is none).",
     "- Each scene has a duration in seconds (typically 3-20) matching how long the shot should last.",
-    "- Apply the same artStyle to every scene imagePrompt.",
-    "- Write prompts and dialogue as natural-language English sentences, never comma-separated keyword tags.",
+].join("\n");
+const SCENE_DETAIL_PROMPT = [
+    "You are a movie pre-production planner. Given a movie idea, its art style, its characters, its places, and one scene, write that scene's full details.",
+    "",
+    "Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
+    "",
+    '{"description":"string","scriptLines":[{"characterSlug":"string","line":"string"}],"voiceOver":"string","imagePrompt":"string"}',
+    "",
+    "Rules:",
+    "- description is a short description of the scene's action.",
+    "- scriptLines is one entry per spoken line, each with the speaking character's slug and the exact spoken line. Include every line of dialogue that belongs to this scene.",
+    "- voiceOver is the narration/voiceover for that scene (use an empty string when there is none).",
+    "- imagePrompt is ONE coherent shot that combines the referenced characters AND the place together, using the given artStyle.",
+    "- Apply the given artStyle to the imagePrompt.",
+    "- Write descriptions, prompts and dialogue as natural-language English sentences, never comma-separated keyword tags.",
 ].join("\n");
 /** Extract the first JSON object from a model response (handles code fences). */
 function extractJsonObject(text) {
@@ -217,7 +226,7 @@ function extractJsonObject(text) {
  * Persists the result to studio/:projectId/data/*.json and returns the parsed
  * result. Used both by the HTTP route below and by the generation queue worker.
  */
-export async function generateMovieStudioBible(projectId, idea, model) {
+export async function generateMovieStudioBible(projectId, idea, model, onProgress) {
     const resolvedModel = typeof model === "string" && model.trim() ? model.trim() : DEFAULT_MODEL;
     const client = new OpenAI({
         baseURL: "http://localhost:8881/v1",
@@ -229,51 +238,127 @@ export async function generateMovieStudioBible(projectId, idea, model) {
         return Number.isFinite(n) ? n : 0;
     };
     const ask = async (systemPrompt, userContent) => {
-        const completion = await client.chat.completions.create({
-            model: resolvedModel,
-            messages: [
-                { role: "system", content: systemPrompt },
-                { role: "user", content: userContent },
-            ],
-            temperature: 0.7,
-        });
-        const text = completion.choices?.[0]?.message?.content ?? "";
-        return extractJsonObject(text);
+        let prompt = systemPrompt;
+        let lastError = null;
+        for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+                const completion = await client.chat.completions.create({
+                    model: resolvedModel,
+                    messages: [
+                        { role: "system", content: prompt },
+                        { role: "user", content: userContent },
+                    ],
+                    temperature: 0.7,
+                    max_tokens: 8192,
+                });
+                const text = completion.choices?.[0]?.message?.content ?? "";
+                return extractJsonObject(text);
+            }
+            catch (e) {
+                lastError = e;
+                // The model likely truncated its JSON — retry with an explicit
+                // instruction to emit one complete, valid JSON object.
+                prompt = `${systemPrompt}\n\nIMPORTANT: Return the complete JSON object in a single response. Do not truncate; close every string, object, and array.`;
+            }
+        }
+        throw lastError instanceof Error
+            ? lastError
+            : new Error("Failed to parse model JSON response");
+    };
+    // Generate one item through the model, falling back to `fallback` on failure
+    // (after the ask-level retries) so a single bad item never fails the whole
+    // production bible.
+    const generateItem = async (systemPrompt, userContent, fallback) => {
+        try {
+            return await ask(systemPrompt, userContent);
+        }
+        catch {
+            return fallback;
+        }
+    };
+    // Progress reporting (SSE-synced to the UI by the queue worker).
+    let current = 0;
+    let total = 0;
+    const addTotal = (n) => {
+        total += n;
     };
-    // 1. Characters + art style.
-    const charData = await ask(MOVIE_STUDIO_CHARACTERS_PROMPT, idea.trim());
-    const artStyle = toStr(charData?.artStyle);
-    const characters = (Array.isArray(charData.characters) ? charData.characters : []).map((c) => ({
-        slug: toStr(c?.slug),
-        name: toStr(c?.name),
-        imagePrompt: toStr(c?.imagePrompt),
-    }));
-    // 2. Places (with art style + characters as context).
-    const placeData = await ask(MOVIE_STUDIO_PLACES_PROMPT, `Art style: ${artStyle || "photo realistic render"}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters)}`);
-    const places = (Array.isArray(placeData.places) ? placeData.places : []).map((p) => ({
-        slug: toStr(p?.slug),
-        name: toStr(p?.name),
-        imagePrompt: toStr(p?.imagePrompt),
-    }));
-    // 3. Scenes (with art style + characters + places as context).
-    const sceneData = await ask(MOVIE_STUDIO_SCENES_PROMPT, `Art style: ${artStyle || "photo realistic render"}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters)}\n\nPlaces:\n${JSON.stringify(places)}`);
-    const scenes = (Array.isArray(sceneData.scenes) ? sceneData.scenes : []).map((s) => ({
+    const note = (statusText) => onProgress?.(statusText, current, total);
+    const advance = (statusText) => {
+        current += 1;
+        onProgress?.(statusText, current, total);
+    };
+    // 1. Character plan — a compact list of slugs/names + the film's art style.
+    note("Identifying characters…");
+    const charPlan = await ask(MOVIE_STUDIO_CHARACTERS_PLAN_PROMPT, idea.trim());
+    const artStyle = toStr(charPlan?.artStyle) || "photo realistic render";
+    const charList = (Array.isArray(charPlan.characters) ? charPlan.characters : [])
+        .map((c) => ({ slug: toStr(c?.slug), name: toStr(c?.name) }))
+        .filter((c) => c.slug);
+    addTotal(charList.length);
+    // 2. Character details — one image prompt per character (agentic loop).
+    const characters = [];
+    for (const c of charList) {
+        advance(`Generating character image: ${c.name || c.slug}`);
+        const detail = await generateItem(CHARACTER_IMAGE_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacter slug: ${c.slug}\nCharacter name: ${c.name}`, { imagePrompt: "" });
+        characters.push({
+            slug: c.slug,
+            name: c.name,
+            imagePrompt: toStr(detail?.imagePrompt),
+        });
+    }
+    // 3. Place plan + details.
+    note("Identifying places…");
+    const placePlan = await ask(MOVIE_STUDIO_PLACES_PLAN_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters.map((c) => ({ slug: c.slug, name: c.name })))}`);
+    const placeList = (Array.isArray(placePlan.places) ? placePlan.places : [])
+        .map((p) => ({ slug: toStr(p?.slug), name: toStr(p?.name) }))
+        .filter((p) => p.slug);
+    addTotal(placeList.length);
+    const places = [];
+    for (const p of placeList) {
+        advance(`Generating place image: ${p.name || p.slug}`);
+        const detail = await generateItem(PLACE_IMAGE_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nPlace slug: ${p.slug}\nPlace name: ${p.name}`, { imagePrompt: "" });
+        places.push({
+            slug: p.slug,
+            name: p.name,
+            imagePrompt: toStr(detail?.imagePrompt),
+        });
+    }
+    // 4. Scene plan + details — one scene's full details per request.
+    note("Planning scenes…");
+    const scenePlan = await ask(MOVIE_STUDIO_SCENES_PLAN_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters.map((c) => ({ slug: c.slug, name: c.name })))}\n\nPlaces:\n${JSON.stringify(places.map((p) => ({ slug: p.slug, name: p.name })))}`);
+    const sceneList = (Array.isArray(scenePlan.scenes) ? scenePlan.scenes : [])
+        .map((s) => ({
         slug: toStr(s?.slug),
         duration: toNum(s?.duration),
-        description: toStr(s?.description),
         characterSlugs: Array.isArray(s?.characterSlugs)
             ? s.characterSlugs.map(toStr).filter(Boolean)
             : [],
         placeSlug: toStr(s?.placeSlug),
-        scriptLines: Array.isArray(s?.scriptLines)
-            ? s.scriptLines.map((l) => ({
-                characterSlug: toStr(l?.characterSlug),
-                line: toStr(l?.line),
-            }))
-            : [],
-        voiceOver: toStr(s?.voiceOver),
-        imagePrompt: toStr(s?.imagePrompt),
-    }));
+    }))
+        .filter((s) => s.slug);
+    addTotal(sceneList.length);
+    const scenes = [];
+    for (const s of sceneList) {
+        advance(`Writing scene: ${s.slug}`);
+        const detail = await generateItem(SCENE_DETAIL_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters.map((c) => ({ slug: c.slug, name: c.name })))}\n\nPlaces:\n${JSON.stringify(places.map((p) => ({ slug: p.slug, name: p.name })))}\n\nScene slug: ${s.slug}\nDuration: ${s.duration}s\nCharacter slugs: ${s.characterSlugs.join(", ")}\nPlace slug: ${s.placeSlug}`, { description: "", scriptLines: [], voiceOver: "", imagePrompt: "" });
+        scenes.push({
+            slug: s.slug,
+            duration: s.duration,
+            description: toStr(detail?.description),
+            characterSlugs: s.characterSlugs,
+            placeSlug: s.placeSlug,
+            scriptLines: Array.isArray(detail?.scriptLines)
+                ? detail.scriptLines
+                    .map((l) => ({
+                    characterSlug: toStr(l?.characterSlug),
+                    line: toStr(l?.line),
+                }))
+                    .filter((l) => l.line)
+                : [],
+            voiceOver: toStr(detail?.voiceOver),
+            imagePrompt: toStr(detail?.imagePrompt),
+        });
+    }
     // Persist the generated production bible to studio/:projectId/data/*.json
     const dataDir = movieStudioDataDir(projectId);
     ensureDir(dataDir);
@@ -684,7 +769,7 @@ export async function agentBackend({ app, getUvPath: _getUvPath, backendPort, })
         }
     });
     app.post("/api/movie-studio/state", (req, res) => {
-        const { projectId, idea, result, assets, videos, sceneImages } = req.body || {};
+        const { projectId, idea, result, assets, videos, sceneImages, renderedScenes, } = req.body || {};
         if (!projectId || !/^[a-zA-Z0-9_-]{1,64}$/.test(String(projectId))) {
             res.status(400).json({ error: "Invalid project ID" });
             return;
@@ -697,6 +782,7 @@ export async function agentBackend({ app, getUvPath: _getUvPath, backendPort, })
             assets: Array.isArray(assets) ? assets : [],
             videos: Array.isArray(videos) ? videos : [],
             sceneImages: Array.isArray(sceneImages) ? sceneImages : [],
+            renderedScenes: Array.isArray(renderedScenes) ? renderedScenes : [],
         }, null, 2), "utf-8");
         res.json({ ok: true });
     });

package/dist/backend/movie-backend/agent/tools/index.js

@@ -7,7 +7,6 @@ import removeFileTool from "./remove-file.js";
 import renameFileTool from "./rename-file.js";
 import grepFilesTool from "./grep-files.js";
 import showImageTool from "./show-image.js";
-import editImageTool from "./edit-image.js";
 import imageToVideoGenerationTool from "./image-to-video-generation.js";
 import textToVideoGenerationTool from "./text-to-video-generation.js";
 import stitchVideosTool from "./stitch-videos.js";
@@ -21,7 +20,6 @@ export const TOOLS = [
     renameFileTool,
     grepFilesTool,
     showImageTool,
-    editImageTool,
     imageToVideoGenerationTool,
     textToVideoGenerationTool,
     stitchVideosTool,

package/dist/backend/movie-backend/core.js

@@ -866,7 +866,7 @@ async function streamProcessOutput(readable, prefix, send) {
 //       "--model",
 //       "dgrauet/ltx-2.3-mlx-q4",
 //       "--prompt",
-//       `${JSON.stringify("No text on screen. a 5 years old cute lamb wanting to have a hug, he says: Hi! Thank you for using Lambobo Studio!")}`,
+//       `${JSON.stringify("No text on screen. a 5 years old cute lamb wanting to have a hug, he says: Hi! Thank you for using EffectNode Media!")}`,
 //       "--distilled",
 //       "--low-ram",
 //       "--frames",

package/dist/backend/movie-backend/generation-queue.js

@@ -379,7 +379,9 @@ const handlers = {
             throw new Error("Idea is required");
         }
         ctx.log(`Planning production bible (characters → places → scenes)…\n`);
-        const result = await generateMovieStudioBible(ctx.projectId, idea.trim(), typeof model === "string" && model.trim() ? model.trim() : undefined);
+        const result = await generateMovieStudioBible(ctx.projectId, idea.trim(), typeof model === "string" && model.trim() ? model.trim() : undefined, (statusText, current, total) => {
+            ctx.update({ statusText, progress: { current, total } });
+        });
         ctx.log("Production bible ready.\n");
         return result;
     },
@@ -574,6 +576,11 @@ function syncToMovieStudioState(projectId, type, result) {
     if (type === "render-videos" || type === "render") {
         state.videos = Array.isArray(result?.videos) ? result.videos : state.videos ?? [];
     }
+    if (type === "render") {
+        state.renderedScenes = Array.isArray(result?.renderedScenes)
+            ? result.renderedScenes
+            : state.renderedScenes ?? [];
+    }
     if (type === "regenerate-asset" && result) {
         const key = `${result.kind}:${result.slug}`;
         const arr = Array.isArray(state.assets) ? state.assets : [];