@effectnode/media 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/backend/movie-backend/agent/agent-backend.d.ts +1 -1
- package/dist/backend/movie-backend/agent/agent-backend.js +172 -83
- package/dist/backend/movie-backend/agent/tools/index.js +0 -2
- package/dist/backend/movie-backend/core.js +1 -1
- package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
- package/dist/backend/movie-backend/generation-queue.js +128 -9
- package/dist/backend/movie-backend/render-media.js +49 -226
- package/frontend/index.html +3 -5
- package/frontend/src/movie-app/SetupPage.tsx +1 -1
- package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +127 -23
- package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +38 -37
- package/frontend/src/movie-app/components/EditorTabs/TaskQueuePanel.tsx +37 -9
- package/frontend/src/movie-app/components/ProjectEditorPage.tsx +0 -33
- package/frontend/src/movie-app/components/ProjectManager.tsx +1 -1
- package/frontend/src/movie-app/index.css +46 -20
- package/frontend/src/movie-app/index.html +1 -1
- package/frontend/src/movie-app/stores/aiModelStore.ts +20 -7
- package/frontend/src/movie-app/stores/generationStore.ts +0 -1
- package/frontend/src/movie-app/stores/movieStudioStore.ts +359 -32
- package/frontend/src/movie-app/stores/queueStore.ts +66 -1
- package/package.json +2 -2
- package/frontend/src/movie-app/components/EditorTabs/CharactersTab.tsx +0 -664
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 @wonglok831 x.com, github.com/wonglok
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
@@ -4,7 +4,7 @@ import { type Application } from "express";
|
|
|
4
4
|
* Persists the result to studio/:projectId/data/*.json and returns the parsed
|
|
5
5
|
* result. Used both by the HTTP route below and by the generation queue worker.
|
|
6
6
|
*/
|
|
7
|
-
export declare function generateMovieStudioBible(projectId: string, idea: string, model?: string): Promise<{
|
|
7
|
+
export declare function generateMovieStudioBible(projectId: string, idea: string, model?: string, onProgress?: (statusText: string, current: number, total: number) => void): Promise<{
|
|
8
8
|
characters: any[];
|
|
9
9
|
places: any[];
|
|
10
10
|
scenes: any[];
|
|
@@ -48,7 +48,7 @@ function toOpenAIMessages(messages) {
|
|
|
48
48
|
function buildSystemPrompt() {
|
|
49
49
|
const toolList = TOOLS.map((t) => `- ${t.name}: ${t.description}`).join("\n");
|
|
50
50
|
return [
|
|
51
|
-
"You are a media assistant that helps the user
|
|
51
|
+
"You are a media assistant that helps the user render videos in their workspace.",
|
|
52
52
|
"",
|
|
53
53
|
"Available tools:",
|
|
54
54
|
toolList,
|
|
@@ -56,34 +56,16 @@ function buildSystemPrompt() {
|
|
|
56
56
|
"Rules:",
|
|
57
57
|
"- Call a tool only when you actually need its result. Once you have enough information, answer the user directly WITHOUT calling any tool.",
|
|
58
58
|
"- Do not call the same tool twice with the same arguments.",
|
|
59
|
-
"- ALWAYS confirm with the user before generating
|
|
59
|
+
"- ALWAYS confirm with the user before generating. Never call `image_to_video_generation` or `text_to_video_generation` until the user has explicitly said yes.",
|
|
60
60
|
"",
|
|
61
61
|
"## Getting started",
|
|
62
62
|
"",
|
|
63
63
|
'When the user greets you (e.g. says "hi" or "hello") or asks for help, FIRST ask them what they want to do:',
|
|
64
|
-
"1.
|
|
65
|
-
"2. Render a video from
|
|
66
|
-
"3. Render a video from an image",
|
|
64
|
+
"1. Render a video from text",
|
|
65
|
+
"2. Render a video from an image",
|
|
67
66
|
"",
|
|
68
67
|
"Then follow the matching workflow below.",
|
|
69
68
|
"",
|
|
70
|
-
"## Workflow: Edit an image",
|
|
71
|
-
"",
|
|
72
|
-
"### Step 1 — Image",
|
|
73
|
-
"- Call `list_files` to see what is in the agent workspace.",
|
|
74
|
-
'- List every image file (files whose `kind` is "image") to the user as a numbered list so they can pick one.',
|
|
75
|
-
"- You may call `show_image` to preview an image before the user decides.",
|
|
76
|
-
"- If there are no images, tell the user and ask them to upload one first.",
|
|
77
|
-
"",
|
|
78
|
-
"### Step 2 — Edit instruction",
|
|
79
|
-
"- Plan the edit together with the user. Ask what they want to change: background, style, color, lighting, expression, object, and so on.",
|
|
80
|
-
"- Confirm the wording of the edit prompt before moving on.",
|
|
81
|
-
"",
|
|
82
|
-
"### Step 3 — Summary and confirmation",
|
|
83
|
-
"- Prepare a short summary containing the image and the edit instruction, and ask the user to confirm.",
|
|
84
|
-
"- Do NOT start editing yet. Wait for the user to say yes.",
|
|
85
|
-
"- Only after the user confirms, call `edit_image` with the image and prompt.",
|
|
86
|
-
"",
|
|
87
69
|
"## Workflow: Render a video",
|
|
88
70
|
"",
|
|
89
71
|
"To generate a video you need a motion PROMPT and a DURATION. An IMAGE is optional but recommended.",
|
|
@@ -151,53 +133,83 @@ function sanitizeTrace(messages) {
|
|
|
151
133
|
});
|
|
152
134
|
}
|
|
153
135
|
// ========== Movie Studio ==========
|
|
154
|
-
const
|
|
136
|
+
const MOVIE_STUDIO_CHARACTERS_PLAN_PROMPT = [
|
|
155
137
|
"You are a movie pre-production planner. Given a movie or story idea, identify the characters.",
|
|
156
138
|
"",
|
|
157
139
|
"Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
|
|
158
140
|
"",
|
|
159
|
-
'{"artStyle":"string","characters":[{"slug":"string","name":"string"
|
|
141
|
+
'{"artStyle":"string","characters":[{"slug":"string","name":"string"}]}',
|
|
160
142
|
"",
|
|
161
143
|
"Rules:",
|
|
162
144
|
'- "slug" is a short lowercase hyphenated identifier (e.g. "the-lamb").',
|
|
163
145
|
'- "artStyle" is ONE consistent art style for the entire film (e.g. cinematic photorealistic, 3D animation, hand-painted watercolor, anime).',
|
|
164
146
|
'- If the idea does not mention an art style, set "artStyle" to "photo realistic render".',
|
|
165
|
-
"- A character's imagePrompt is a standalone text-to-image prompt that begins with the character's name and fully describes their appearance (age, face, build, outfit, distinctive features) together with their cultural background — the era/age, culture, ethnicity, and region the character belongs to — so the generated image is historically and culturally accurate and consistent.",
|
|
166
|
-
"- Apply the same artStyle to every character imagePrompt.",
|
|
167
|
-
"- Write prompts as natural-language English sentences, never comma-separated keyword tags.",
|
|
168
147
|
].join("\n");
|
|
169
|
-
const
|
|
148
|
+
const CHARACTER_IMAGE_PROMPT = [
|
|
149
|
+
"You are a movie pre-production planner. Given a movie idea, its art style, and one character, write that character's standalone character-reference image prompt.",
|
|
150
|
+
"",
|
|
151
|
+
"Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
|
|
152
|
+
"",
|
|
153
|
+
'{"imagePrompt":"string"}',
|
|
154
|
+
"",
|
|
155
|
+
"Rules:",
|
|
156
|
+
"- The imagePrompt describes a close-up portrait shot of the character's face on a plain solid white background, framed head and shoulders, facing the camera, evenly lit with soft neutral studio lighting and no props, scenery or background elements.",
|
|
157
|
+
"- The imagePrompt begins with the character's name and then concentrates on the face: age, face shape, skin tone and texture, eyes, eyebrows, nose, mouth, facial hair, hairstyle and hair colour, expression, and any distinctive facial features such as scars, freckles or markings.",
|
|
158
|
+
"- Include the character's cultural background — the era/age, culture, ethnicity, and region they belong to — so the face is historically and culturally accurate and consistent.",
|
|
159
|
+
"- Mention clothing only as far as the collar or neckline is visible in a head-and-shoulders crop; do not describe the full outfit, body build, pose, or action.",
|
|
160
|
+
"- Apply the given artStyle.",
|
|
161
|
+
"- Write the prompt as natural-language English sentences, never comma-separated keyword tags.",
|
|
162
|
+
].join("\n");
|
|
163
|
+
const MOVIE_STUDIO_PLACES_PLAN_PROMPT = [
|
|
170
164
|
"You are a movie pre-production planner. Given a movie idea, its art style, and its characters, identify the places/locations.",
|
|
171
165
|
"",
|
|
172
166
|
"Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
|
|
173
167
|
"",
|
|
174
|
-
'{"places":[{"slug":"string","name":"string"
|
|
168
|
+
'{"places":[{"slug":"string","name":"string"}]}',
|
|
175
169
|
"",
|
|
176
170
|
"Rules:",
|
|
177
171
|
'- "slug" is a short lowercase hyphenated identifier (e.g. "sunny-meadow").',
|
|
178
|
-
"- A place's imagePrompt is a standalone text-to-image prompt that fully describes the location/environment (time of day, lighting, atmosphere, visual style).",
|
|
179
|
-
"- Apply the given artStyle to every place imagePrompt so it matches the characters.",
|
|
180
|
-
"- Write prompts as natural-language English sentences, never comma-separated keyword tags.",
|
|
181
172
|
].join("\n");
|
|
182
|
-
const
|
|
173
|
+
const PLACE_IMAGE_PROMPT = [
|
|
174
|
+
"You are a movie pre-production planner. Given a movie idea, its art style, and one place, write that place's standalone image prompt.",
|
|
175
|
+
"",
|
|
176
|
+
"Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
|
|
177
|
+
"",
|
|
178
|
+
'{"imagePrompt":"string"}',
|
|
179
|
+
"",
|
|
180
|
+
"Rules:",
|
|
181
|
+
"- The imagePrompt fully describes the location/environment (time of day, lighting, atmosphere, visual style).",
|
|
182
|
+
"- Apply the given artStyle.",
|
|
183
|
+
"- Write the prompt as natural-language English sentences, never comma-separated keyword tags.",
|
|
184
|
+
].join("\n");
|
|
185
|
+
const MOVIE_STUDIO_SCENES_PLAN_PROMPT = [
|
|
183
186
|
"You are a movie pre-production planner. Given a movie idea, its art style, its characters, and its places, break the story into scenes.",
|
|
184
187
|
"",
|
|
185
188
|
"Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
|
|
186
189
|
"",
|
|
187
|
-
'{"scenes":[{"slug":"string","duration":number,"
|
|
190
|
+
'{"scenes":[{"slug":"string","duration":number,"characterSlugs":["string"],"placeSlug":"string"}]}',
|
|
188
191
|
"",
|
|
189
192
|
"Rules:",
|
|
190
|
-
|
|
191
|
-
"-
|
|
192
|
-
"- Never reduce or summarize the dialogue:
|
|
193
|
-
"- Preserve as much detail as possible in every description and imagePrompt.",
|
|
193
|
+
'- "slug" is a short lowercase hyphenated identifier (e.g. "opening-meadow").',
|
|
194
|
+
"- Break the story into a natural number of scenes. Group related action beats and dialogue together — do not inflate the scene count by splitting every moment into its own scene.",
|
|
195
|
+
"- Never reduce or summarize the dialogue: keep every spoken line, grouping multiple lines into a scene where it reads naturally.",
|
|
194
196
|
"- Each scene references the characters and the place involved via their slugs (characterSlugs is a list; placeSlug is a single slug).",
|
|
195
|
-
"- A scene's imagePrompt is ONE coherent shot that combines the referenced characters AND the place together, using the given artStyle.",
|
|
196
|
-
"- scriptLines is one entry per spoken line, each with the speaking character's slug and the exact spoken line. Include every single line of dialogue in the scene.",
|
|
197
|
-
"- voiceOver is the narration/voiceover for that scene (use an empty string when there is none).",
|
|
198
197
|
"- Each scene has a duration in seconds (typically 3-20) matching how long the shot should last.",
|
|
199
|
-
|
|
200
|
-
|
|
198
|
+
].join("\n");
|
|
199
|
+
const SCENE_DETAIL_PROMPT = [
|
|
200
|
+
"You are a movie pre-production planner. Given a movie idea, its art style, its characters, its places, and one scene, write that scene's full details.",
|
|
201
|
+
"",
|
|
202
|
+
"Return ONLY valid JSON (no markdown fences, no commentary) matching this exact shape:",
|
|
203
|
+
"",
|
|
204
|
+
'{"description":"string","scriptLines":[{"characterSlug":"string","line":"string"}],"voiceOver":"string","imagePrompt":"string"}',
|
|
205
|
+
"",
|
|
206
|
+
"Rules:",
|
|
207
|
+
"- description is a short description of the scene's action.",
|
|
208
|
+
"- scriptLines is one entry per spoken line, each with the speaking character's slug and the exact spoken line. Include every line of dialogue that belongs to this scene.",
|
|
209
|
+
"- voiceOver is the narration/voiceover for that scene (use an empty string when there is none).",
|
|
210
|
+
"- imagePrompt is ONE coherent shot that combines the referenced characters AND the place together, using the given artStyle.",
|
|
211
|
+
"- Apply the given artStyle to the imagePrompt.",
|
|
212
|
+
"- Write descriptions, prompts and dialogue as natural-language English sentences, never comma-separated keyword tags.",
|
|
201
213
|
].join("\n");
|
|
202
214
|
/** Extract the first JSON object from a model response (handles code fences). */
|
|
203
215
|
function extractJsonObject(text) {
|
|
@@ -217,7 +229,7 @@ function extractJsonObject(text) {
|
|
|
217
229
|
* Persists the result to studio/:projectId/data/*.json and returns the parsed
|
|
218
230
|
* result. Used both by the HTTP route below and by the generation queue worker.
|
|
219
231
|
*/
|
|
220
|
-
export async function generateMovieStudioBible(projectId, idea, model) {
|
|
232
|
+
export async function generateMovieStudioBible(projectId, idea, model, onProgress) {
|
|
221
233
|
const resolvedModel = typeof model === "string" && model.trim() ? model.trim() : DEFAULT_MODEL;
|
|
222
234
|
const client = new OpenAI({
|
|
223
235
|
baseURL: "http://localhost:8881/v1",
|
|
@@ -229,51 +241,127 @@ export async function generateMovieStudioBible(projectId, idea, model) {
|
|
|
229
241
|
return Number.isFinite(n) ? n : 0;
|
|
230
242
|
};
|
|
231
243
|
const ask = async (systemPrompt, userContent) => {
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
244
|
+
let prompt = systemPrompt;
|
|
245
|
+
let lastError = null;
|
|
246
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
247
|
+
try {
|
|
248
|
+
const completion = await client.chat.completions.create({
|
|
249
|
+
model: resolvedModel,
|
|
250
|
+
messages: [
|
|
251
|
+
{ role: "system", content: prompt },
|
|
252
|
+
{ role: "user", content: userContent },
|
|
253
|
+
],
|
|
254
|
+
temperature: 0.7,
|
|
255
|
+
max_tokens: 8192,
|
|
256
|
+
});
|
|
257
|
+
const text = completion.choices?.[0]?.message?.content ?? "";
|
|
258
|
+
return extractJsonObject(text);
|
|
259
|
+
}
|
|
260
|
+
catch (e) {
|
|
261
|
+
lastError = e;
|
|
262
|
+
// The model likely truncated its JSON — retry with an explicit
|
|
263
|
+
// instruction to emit one complete, valid JSON object.
|
|
264
|
+
prompt = `${systemPrompt}\n\nIMPORTANT: Return the complete JSON object in a single response. Do not truncate; close every string, object, and array.`;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
throw lastError instanceof Error
|
|
268
|
+
? lastError
|
|
269
|
+
: new Error("Failed to parse model JSON response");
|
|
270
|
+
};
|
|
271
|
+
// Generate one item through the model, falling back to `fallback` on failure
|
|
272
|
+
// (after the ask-level retries) so a single bad item never fails the whole
|
|
273
|
+
// production bible.
|
|
274
|
+
const generateItem = async (systemPrompt, userContent, fallback) => {
|
|
275
|
+
try {
|
|
276
|
+
return await ask(systemPrompt, userContent);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
return fallback;
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
// Progress reporting (SSE-synced to the UI by the queue worker).
|
|
283
|
+
let current = 0;
|
|
284
|
+
let total = 0;
|
|
285
|
+
const addTotal = (n) => {
|
|
286
|
+
total += n;
|
|
242
287
|
};
|
|
243
|
-
|
|
244
|
-
const
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
const
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
288
|
+
const note = (statusText) => onProgress?.(statusText, current, total);
|
|
289
|
+
const advance = (statusText) => {
|
|
290
|
+
current += 1;
|
|
291
|
+
onProgress?.(statusText, current, total);
|
|
292
|
+
};
|
|
293
|
+
// 1. Character plan — a compact list of slugs/names + the film's art style.
|
|
294
|
+
note("Identifying characters…");
|
|
295
|
+
const charPlan = await ask(MOVIE_STUDIO_CHARACTERS_PLAN_PROMPT, idea.trim());
|
|
296
|
+
const artStyle = toStr(charPlan?.artStyle) || "photo realistic render";
|
|
297
|
+
const charList = (Array.isArray(charPlan.characters) ? charPlan.characters : [])
|
|
298
|
+
.map((c) => ({ slug: toStr(c?.slug), name: toStr(c?.name) }))
|
|
299
|
+
.filter((c) => c.slug);
|
|
300
|
+
addTotal(charList.length);
|
|
301
|
+
// 2. Character details — one image prompt per character (agentic loop).
|
|
302
|
+
const characters = [];
|
|
303
|
+
for (const c of charList) {
|
|
304
|
+
advance(`Generating character image: ${c.name || c.slug}`);
|
|
305
|
+
const detail = await generateItem(CHARACTER_IMAGE_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacter slug: ${c.slug}\nCharacter name: ${c.name}`, { imagePrompt: "" });
|
|
306
|
+
characters.push({
|
|
307
|
+
slug: c.slug,
|
|
308
|
+
name: c.name,
|
|
309
|
+
imagePrompt: toStr(detail?.imagePrompt),
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
// 3. Place plan + details.
|
|
313
|
+
note("Identifying places…");
|
|
314
|
+
const placePlan = await ask(MOVIE_STUDIO_PLACES_PLAN_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters.map((c) => ({ slug: c.slug, name: c.name })))}`);
|
|
315
|
+
const placeList = (Array.isArray(placePlan.places) ? placePlan.places : [])
|
|
316
|
+
.map((p) => ({ slug: toStr(p?.slug), name: toStr(p?.name) }))
|
|
317
|
+
.filter((p) => p.slug);
|
|
318
|
+
addTotal(placeList.length);
|
|
319
|
+
const places = [];
|
|
320
|
+
for (const p of placeList) {
|
|
321
|
+
advance(`Generating place image: ${p.name || p.slug}`);
|
|
322
|
+
const detail = await generateItem(PLACE_IMAGE_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nPlace slug: ${p.slug}\nPlace name: ${p.name}`, { imagePrompt: "" });
|
|
323
|
+
places.push({
|
|
324
|
+
slug: p.slug,
|
|
325
|
+
name: p.name,
|
|
326
|
+
imagePrompt: toStr(detail?.imagePrompt),
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
// 4. Scene plan + details — one scene's full details per request.
|
|
330
|
+
note("Planning scenes…");
|
|
331
|
+
const scenePlan = await ask(MOVIE_STUDIO_SCENES_PLAN_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters.map((c) => ({ slug: c.slug, name: c.name })))}\n\nPlaces:\n${JSON.stringify(places.map((p) => ({ slug: p.slug, name: p.name })))}`);
|
|
332
|
+
const sceneList = (Array.isArray(scenePlan.scenes) ? scenePlan.scenes : [])
|
|
333
|
+
.map((s) => ({
|
|
261
334
|
slug: toStr(s?.slug),
|
|
262
335
|
duration: toNum(s?.duration),
|
|
263
|
-
description: toStr(s?.description),
|
|
264
336
|
characterSlugs: Array.isArray(s?.characterSlugs)
|
|
265
337
|
? s.characterSlugs.map(toStr).filter(Boolean)
|
|
266
338
|
: [],
|
|
267
339
|
placeSlug: toStr(s?.placeSlug),
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
340
|
+
}))
|
|
341
|
+
.filter((s) => s.slug);
|
|
342
|
+
addTotal(sceneList.length);
|
|
343
|
+
const scenes = [];
|
|
344
|
+
for (const s of sceneList) {
|
|
345
|
+
advance(`Writing scene: ${s.slug}`);
|
|
346
|
+
const detail = await generateItem(SCENE_DETAIL_PROMPT, `Art style: ${artStyle}\n\nIdea: ${idea.trim()}\n\nCharacters:\n${JSON.stringify(characters.map((c) => ({ slug: c.slug, name: c.name })))}\n\nPlaces:\n${JSON.stringify(places.map((p) => ({ slug: p.slug, name: p.name })))}\n\nScene slug: ${s.slug}\nDuration: ${s.duration}s\nCharacter slugs: ${s.characterSlugs.join(", ")}\nPlace slug: ${s.placeSlug}`, { description: "", scriptLines: [], voiceOver: "", imagePrompt: "" });
|
|
347
|
+
scenes.push({
|
|
348
|
+
slug: s.slug,
|
|
349
|
+
duration: s.duration,
|
|
350
|
+
description: toStr(detail?.description),
|
|
351
|
+
characterSlugs: s.characterSlugs,
|
|
352
|
+
placeSlug: s.placeSlug,
|
|
353
|
+
scriptLines: Array.isArray(detail?.scriptLines)
|
|
354
|
+
? detail.scriptLines
|
|
355
|
+
.map((l) => ({
|
|
356
|
+
characterSlug: toStr(l?.characterSlug),
|
|
357
|
+
line: toStr(l?.line),
|
|
358
|
+
}))
|
|
359
|
+
.filter((l) => l.line)
|
|
360
|
+
: [],
|
|
361
|
+
voiceOver: toStr(detail?.voiceOver),
|
|
362
|
+
imagePrompt: toStr(detail?.imagePrompt),
|
|
363
|
+
});
|
|
364
|
+
}
|
|
277
365
|
// Persist the generated production bible to studio/:projectId/data/*.json
|
|
278
366
|
const dataDir = movieStudioDataDir(projectId);
|
|
279
367
|
ensureDir(dataDir);
|
|
@@ -684,7 +772,7 @@ export async function agentBackend({ app, getUvPath: _getUvPath, backendPort, })
|
|
|
684
772
|
}
|
|
685
773
|
});
|
|
686
774
|
app.post("/api/movie-studio/state", (req, res) => {
|
|
687
|
-
const { projectId, idea, result, assets, videos, sceneImages } = req.body || {};
|
|
775
|
+
const { projectId, idea, result, assets, videos, sceneImages, renderedScenes, } = req.body || {};
|
|
688
776
|
if (!projectId || !/^[a-zA-Z0-9_-]{1,64}$/.test(String(projectId))) {
|
|
689
777
|
res.status(400).json({ error: "Invalid project ID" });
|
|
690
778
|
return;
|
|
@@ -697,6 +785,7 @@ export async function agentBackend({ app, getUvPath: _getUvPath, backendPort, })
|
|
|
697
785
|
assets: Array.isArray(assets) ? assets : [],
|
|
698
786
|
videos: Array.isArray(videos) ? videos : [],
|
|
699
787
|
sceneImages: Array.isArray(sceneImages) ? sceneImages : [],
|
|
788
|
+
renderedScenes: Array.isArray(renderedScenes) ? renderedScenes : [],
|
|
700
789
|
}, null, 2), "utf-8");
|
|
701
790
|
res.json({ ok: true });
|
|
702
791
|
});
|
|
@@ -7,7 +7,6 @@ import removeFileTool from "./remove-file.js";
|
|
|
7
7
|
import renameFileTool from "./rename-file.js";
|
|
8
8
|
import grepFilesTool from "./grep-files.js";
|
|
9
9
|
import showImageTool from "./show-image.js";
|
|
10
|
-
import editImageTool from "./edit-image.js";
|
|
11
10
|
import imageToVideoGenerationTool from "./image-to-video-generation.js";
|
|
12
11
|
import textToVideoGenerationTool from "./text-to-video-generation.js";
|
|
13
12
|
import stitchVideosTool from "./stitch-videos.js";
|
|
@@ -21,7 +20,6 @@ export const TOOLS = [
|
|
|
21
20
|
renameFileTool,
|
|
22
21
|
grepFilesTool,
|
|
23
22
|
showImageTool,
|
|
24
|
-
editImageTool,
|
|
25
23
|
imageToVideoGenerationTool,
|
|
26
24
|
textToVideoGenerationTool,
|
|
27
25
|
stitchVideosTool,
|
|
@@ -866,7 +866,7 @@ async function streamProcessOutput(readable, prefix, send) {
|
|
|
866
866
|
// "--model",
|
|
867
867
|
// "dgrauet/ltx-2.3-mlx-q4",
|
|
868
868
|
// "--prompt",
|
|
869
|
-
// `${JSON.stringify("No text on screen. a 5 years old cute lamb wanting to have a hug, he says: Hi! Thank you for using
|
|
869
|
+
// `${JSON.stringify("No text on screen. a 5 years old cute lamb wanting to have a hug, he says: Hi! Thank you for using EffectNode Media!")}`,
|
|
870
870
|
// "--distilled",
|
|
871
871
|
// "--low-ram",
|
|
872
872
|
// "--frames",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type Application } from "express";
|
|
2
|
-
export type QueueTaskType = "generate" | "render" | "render-assets" | "render-videos" | "render-scene-images" | "regenerate-asset" | "regenerate-video" | "regenerate-scene-image";
|
|
2
|
+
export type QueueTaskType = "generate" | "render" | "render-assets" | "render-videos" | "render-scene-images" | "render-asset" | "render-scene-image" | "render-video" | "regenerate-asset" | "regenerate-video" | "regenerate-scene-image";
|
|
3
3
|
export type QueueTaskStatus = "pending" | "running" | "completed" | "failed" | "cancelled" | "paused";
|
|
4
4
|
export interface QueueTask {
|
|
5
5
|
id: string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, } from "node:fs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
@@ -78,10 +78,18 @@ const sseClients = new Set();
|
|
|
78
78
|
/** Push an event to every SSE client watching the given project. */
|
|
79
79
|
function broadcast(projectId, event, data) {
|
|
80
80
|
for (const client of sseClients) {
|
|
81
|
-
if (client.projectId !== projectId)
|
|
81
|
+
if (client.projectId !== null && client.projectId !== projectId)
|
|
82
82
|
continue;
|
|
83
|
+
let payload = data;
|
|
84
|
+
// Global watchers need to know which project each task belongs to.
|
|
85
|
+
if (client.projectId === null &&
|
|
86
|
+
event === "task" &&
|
|
87
|
+
data &&
|
|
88
|
+
typeof data === "object") {
|
|
89
|
+
payload = { ...data, projectId };
|
|
90
|
+
}
|
|
83
91
|
try {
|
|
84
|
-
client.res.write(`event: ${event}\ndata: ${JSON.stringify(
|
|
92
|
+
client.res.write(`event: ${event}\ndata: ${JSON.stringify(payload)}\n\n`);
|
|
85
93
|
}
|
|
86
94
|
catch {
|
|
87
95
|
sseClients.delete(client);
|
|
@@ -130,6 +138,24 @@ function loadState(projectId) {
|
|
|
130
138
|
void pump();
|
|
131
139
|
return state;
|
|
132
140
|
}
|
|
141
|
+
/** Load every persisted project queue into memory so it is visible and resumed. */
|
|
142
|
+
function loadAllStates() {
|
|
143
|
+
let entries = [];
|
|
144
|
+
try {
|
|
145
|
+
entries = existsSync(TASKS_DIR) ? readdirSync(TASKS_DIR) : [];
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
entries = [];
|
|
149
|
+
}
|
|
150
|
+
entries.sort();
|
|
151
|
+
for (const entry of entries) {
|
|
152
|
+
if (!isValidProjectId(entry))
|
|
153
|
+
continue;
|
|
154
|
+
if (!existsSync(queueFile(entry)))
|
|
155
|
+
continue;
|
|
156
|
+
loadState(entry);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
133
159
|
function persist(projectId) {
|
|
134
160
|
const state = queues.get(projectId);
|
|
135
161
|
if (!state)
|
|
@@ -379,7 +405,9 @@ const handlers = {
|
|
|
379
405
|
throw new Error("Idea is required");
|
|
380
406
|
}
|
|
381
407
|
ctx.log(`Planning production bible (characters → places → scenes)…\n`);
|
|
382
|
-
const result = await generateMovieStudioBible(ctx.projectId, idea.trim(), typeof model === "string" && model.trim() ? model.trim() : undefined)
|
|
408
|
+
const result = await generateMovieStudioBible(ctx.projectId, idea.trim(), typeof model === "string" && model.trim() ? model.trim() : undefined, (statusText, current, total) => {
|
|
409
|
+
ctx.update({ statusText, progress: { current, total } });
|
|
410
|
+
});
|
|
383
411
|
ctx.log("Production bible ready.\n");
|
|
384
412
|
return result;
|
|
385
413
|
},
|
|
@@ -398,6 +426,78 @@ const handlers = {
|
|
|
398
426
|
const { characters, places, scenes } = ctx.task.payload || {};
|
|
399
427
|
return runFullRender(ctx, await getUvPath(), characters, places, scenes);
|
|
400
428
|
},
|
|
429
|
+
// Generate a single character/place image, skipping if it already exists.
|
|
430
|
+
"render-asset": async (ctx) => {
|
|
431
|
+
const { kind, slug, prompt } = ctx.task.payload || {};
|
|
432
|
+
if (kind !== "character" && kind !== "place") {
|
|
433
|
+
throw new Error("kind must be 'character' or 'place'");
|
|
434
|
+
}
|
|
435
|
+
const s = slugify(slug);
|
|
436
|
+
const p = String(prompt || "").trim();
|
|
437
|
+
if (!s || !p)
|
|
438
|
+
throw new Error("slug and prompt are required");
|
|
439
|
+
const existing = existingOutput(ctx.projectId, `${kind}-${s}.png`);
|
|
440
|
+
if (existing) {
|
|
441
|
+
ctx.log(`Already generated: ${kind}-${s}.png — skipping\n`);
|
|
442
|
+
return { kind, slug: s, ...existing };
|
|
443
|
+
}
|
|
444
|
+
const r = await generateAssetImage(ctx.projectId, kind, s, p, ctx.log);
|
|
445
|
+
if ("error" in r)
|
|
446
|
+
throw new Error(r.error);
|
|
447
|
+
return {
|
|
448
|
+
kind,
|
|
449
|
+
slug: s,
|
|
450
|
+
filename: r.filename,
|
|
451
|
+
url: r.url,
|
|
452
|
+
updatedAt: Date.now(),
|
|
453
|
+
};
|
|
454
|
+
},
|
|
455
|
+
// Generate a single scene image, skipping if it already exists.
|
|
456
|
+
"render-scene-image": async (ctx) => {
|
|
457
|
+
const { scene } = ctx.task.payload || {};
|
|
458
|
+
if (!scene || typeof scene !== "object")
|
|
459
|
+
throw new Error("scene is required");
|
|
460
|
+
const s = slugify(scene?.slug);
|
|
461
|
+
if (!s)
|
|
462
|
+
throw new Error("Invalid scene slug");
|
|
463
|
+
const existing = existingOutput(ctx.projectId, `scene-${s}.png`);
|
|
464
|
+
if (existing) {
|
|
465
|
+
ctx.log(`Already generated: scene-${s}.png — skipping\n`);
|
|
466
|
+
return { slug: s, ...existing };
|
|
467
|
+
}
|
|
468
|
+
const r = await generateSceneImage(ctx.projectId, scene, ctx.log);
|
|
469
|
+
if ("error" in r)
|
|
470
|
+
throw new Error(r.error);
|
|
471
|
+
return {
|
|
472
|
+
slug: s,
|
|
473
|
+
filename: r.filename,
|
|
474
|
+
url: r.url,
|
|
475
|
+
updatedAt: Date.now(),
|
|
476
|
+
};
|
|
477
|
+
},
|
|
478
|
+
// Generate a single scene video, skipping if it already exists.
|
|
479
|
+
"render-video": async (ctx, getUvPath) => {
|
|
480
|
+
const { scene, characters } = ctx.task.payload || {};
|
|
481
|
+
if (!scene || typeof scene !== "object")
|
|
482
|
+
throw new Error("scene is required");
|
|
483
|
+
const s = slugify(scene?.slug);
|
|
484
|
+
if (!s)
|
|
485
|
+
throw new Error("Invalid scene slug");
|
|
486
|
+
const existing = existingOutput(ctx.projectId, `scene-${s}.mp4`);
|
|
487
|
+
if (existing) {
|
|
488
|
+
ctx.log(`Already generated: scene-${s}.mp4 — skipping\n`);
|
|
489
|
+
return { slug: s, ...existing };
|
|
490
|
+
}
|
|
491
|
+
const r = await generateSceneVideo(await getUvPath(), ctx.projectId, scene, Array.isArray(characters) ? characters : [], ctx.log);
|
|
492
|
+
if ("error" in r)
|
|
493
|
+
throw new Error(r.error);
|
|
494
|
+
return {
|
|
495
|
+
slug: s,
|
|
496
|
+
filename: r.filename,
|
|
497
|
+
url: r.url,
|
|
498
|
+
updatedAt: Date.now(),
|
|
499
|
+
};
|
|
500
|
+
},
|
|
401
501
|
"regenerate-asset": async (ctx) => {
|
|
402
502
|
const { kind, slug, prompt } = ctx.task.payload || {};
|
|
403
503
|
if (kind !== "character" && kind !== "place") {
|
|
@@ -574,7 +674,12 @@ function syncToMovieStudioState(projectId, type, result) {
|
|
|
574
674
|
if (type === "render-videos" || type === "render") {
|
|
575
675
|
state.videos = Array.isArray(result?.videos) ? result.videos : state.videos ?? [];
|
|
576
676
|
}
|
|
577
|
-
if (type === "
|
|
677
|
+
if (type === "render") {
|
|
678
|
+
state.renderedScenes = Array.isArray(result?.renderedScenes)
|
|
679
|
+
? result.renderedScenes
|
|
680
|
+
: state.renderedScenes ?? [];
|
|
681
|
+
}
|
|
682
|
+
if ((type === "render-asset" || type === "regenerate-asset") && result) {
|
|
578
683
|
const key = `${result.kind}:${result.slug}`;
|
|
579
684
|
const arr = Array.isArray(state.assets) ? state.assets : [];
|
|
580
685
|
state.assets = [
|
|
@@ -582,14 +687,15 @@ function syncToMovieStudioState(projectId, type, result) {
|
|
|
582
687
|
result,
|
|
583
688
|
];
|
|
584
689
|
}
|
|
585
|
-
if (type === "regenerate-video" && result) {
|
|
690
|
+
if ((type === "render-video" || type === "regenerate-video") && result) {
|
|
586
691
|
const arr = Array.isArray(state.videos) ? state.videos : [];
|
|
587
692
|
state.videos = [
|
|
588
693
|
...arr.filter((v) => v.slug !== result.slug),
|
|
589
694
|
result,
|
|
590
695
|
];
|
|
591
696
|
}
|
|
592
|
-
if (type === "
|
|
697
|
+
if ((type === "render-scene-image" || type === "regenerate-scene-image") &&
|
|
698
|
+
result) {
|
|
593
699
|
const arr = Array.isArray(state.sceneImages) ? state.sceneImages : [];
|
|
594
700
|
state.sceneImages = [
|
|
595
701
|
...arr.filter((i) => i.slug !== result.slug),
|
|
@@ -638,6 +744,18 @@ export function generationQueueSetup({ app, getUvPath, }) {
|
|
|
638
744
|
paused: pausedProjects.has(projectId),
|
|
639
745
|
});
|
|
640
746
|
});
|
|
747
|
+
// List every task across all projects, each tagged with its project id.
|
|
748
|
+
app.get("/api/queue/all", (_req, res) => {
|
|
749
|
+
loadAllStates();
|
|
750
|
+
const tasks = [];
|
|
751
|
+
for (const [projectId, state] of queues) {
|
|
752
|
+
for (const t of state.tasks) {
|
|
753
|
+
tasks.push({ ...t, projectId });
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
tasks.sort((a, b) => a.createdAt - b.createdAt);
|
|
757
|
+
res.json({ tasks });
|
|
758
|
+
});
|
|
641
759
|
// Read a project's persisted terminal log (tail, so large logs stay bounded).
|
|
642
760
|
app.get("/api/logs", (req, res) => {
|
|
643
761
|
const projectId = String(req.query.projectId ?? "");
|
|
@@ -663,7 +781,8 @@ export function generationQueueSetup({ app, getUvPath, }) {
|
|
|
663
781
|
// Server-Sent Events: push queue/task and log updates to a watching client.
|
|
664
782
|
app.get("/api/events", (req, res) => {
|
|
665
783
|
const projectId = String(req.query.projectId ?? "");
|
|
666
|
-
|
|
784
|
+
const all = projectId === "*";
|
|
785
|
+
if (!all && !isValidProjectId(projectId)) {
|
|
667
786
|
res.status(400).json({ error: "Invalid project ID" });
|
|
668
787
|
return;
|
|
669
788
|
}
|
|
@@ -674,7 +793,7 @@ export function generationQueueSetup({ app, getUvPath, }) {
|
|
|
674
793
|
"X-Accel-Buffering": "no",
|
|
675
794
|
});
|
|
676
795
|
res.write(`event: hello\ndata: {}\n\n`);
|
|
677
|
-
const client = { res, projectId };
|
|
796
|
+
const client = { res, projectId: all ? null : projectId };
|
|
678
797
|
sseClients.add(client);
|
|
679
798
|
req.on("close", () => {
|
|
680
799
|
sseClients.delete(client);
|