@effectnode/media 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/backend/movie-backend/agent/prompt/script.md +213 -0
  2. package/dist/backend/movie-backend/core.js +33 -0
  3. package/dist/backend/movie-backend/generation-queue.d.ts +1 -1
  4. package/dist/backend/movie-backend/generation-queue.js +82 -6
  5. package/dist/backend/movie-backend/render-media.d.ts +89 -1
  6. package/dist/backend/movie-backend/render-media.js +772 -476
  7. package/frontend/src/movie-app/components/EditorTabs/AdvancedVoiceCloneTab.tsx +366 -0
  8. package/frontend/src/movie-app/components/EditorTabs/AudioToVideoTab.tsx +406 -0
  9. package/frontend/src/movie-app/components/EditorTabs/FastImageEditTab.tsx +67 -51
  10. package/frontend/src/movie-app/components/EditorTabs/GenerateVideoTab.tsx +155 -1
  11. package/frontend/src/movie-app/components/EditorTabs/MovieStudioTab.tsx +33 -13
  12. package/frontend/src/movie-app/components/EditorTabs/SetupAiModelTab.tsx +26 -5
  13. package/frontend/src/movie-app/components/EditorTabs/UpscaleTab.tsx +342 -0
  14. package/frontend/src/movie-app/components/EditorTabs/VoiceCloneTab.tsx +365 -0
  15. package/frontend/src/movie-app/components/ProjectEditorPage.tsx +131 -126
  16. package/frontend/src/movie-app/stores/advancedVoiceCloneStore.ts +207 -0
  17. package/frontend/src/movie-app/stores/aiModelStore.ts +25 -2
  18. package/frontend/src/movie-app/stores/audioToVideoStore.ts +274 -0
  19. package/frontend/src/movie-app/stores/generationStore.ts +250 -345
  20. package/frontend/src/movie-app/stores/movieStudioStore.ts +10 -3
  21. package/frontend/src/movie-app/stores/projectStore.ts +7 -3
  22. package/frontend/src/movie-app/stores/queueStore.ts +48 -1
  23. package/frontend/src/movie-app/stores/upscaleStore.ts +118 -0
  24. package/frontend/src/movie-app/stores/voiceCloneStore.ts +227 -0
  25. package/package.json +1 -1
  26. package/frontend/src/movie-app/components/EditorTabs/BatchVoiceVideoTab.tsx +0 -913
  27. package/frontend/src/movie-app/components/EditorTabs/CharacterSheet.tsx +0 -234
  28. package/frontend/src/movie-app/components/EditorTabs/ExtendVideoTab.tsx +0 -305
  29. package/frontend/src/movie-app/components/EditorTabs/ExtractImageTab.tsx +0 -249
  30. package/frontend/src/movie-app/components/EditorTabs/SceneVisualTab.tsx +0 -267
  31. package/frontend/src/movie-app/lib/batchVoiceStorage.ts +0 -75
  32. package/frontend/src/movie-app/stores/batchVoiceStore.ts +0 -990
  33. package/frontend/src/movie-app/stores/sceneVisualStore.ts +0 -251
@@ -1,249 +0,0 @@
1
- import { useEffect, useRef, useState } from "react";
2
- import { useGenerationStore, type ProjectVideo } from "../../stores/generationStore";
3
-
4
- const API_BASE = `http://localhost:${(window as any).PORT}`;
5
-
6
- interface Props {
7
- projectId: string;
8
- }
9
-
10
- /** Seek the video to `time` and draw its current frame into a PNG data URL. */
11
- async function captureFrameAt(
12
- video: HTMLVideoElement,
13
- time: number,
14
- ): Promise<string> {
15
- await new Promise<void>((resolve) => {
16
- if (Math.abs(video.currentTime - time) < 0.001 && !video.seeking) {
17
- resolve();
18
- return;
19
- }
20
- const cleanup = () => {
21
- clearTimeout(timer);
22
- video.removeEventListener("seeked", onSeeked);
23
- };
24
- const onSeeked = () => {
25
- cleanup();
26
- resolve();
27
- };
28
- const timer = setTimeout(onSeeked, 1500);
29
- video.addEventListener("seeked", onSeeked);
30
- video.currentTime = time;
31
- });
32
-
33
- const canvas = document.createElement("canvas");
34
- canvas.width = video.videoWidth;
35
- canvas.height = video.videoHeight;
36
- const ctx = canvas.getContext("2d");
37
- if (!ctx) throw new Error("Canvas not supported");
38
- ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
39
- return canvas.toDataURL("image/png");
40
- }
41
-
42
- export default function ExtractImageTab({ projectId }: Props) {
43
- const gen = useGenerationStore();
44
-
45
- const videoRef = useRef<HTMLVideoElement>(null);
46
-
47
- const [selected, setSelected] = useState<ProjectVideo | null>(null);
48
- const [duration, setDuration] = useState(0);
49
- const [time, setTime] = useState(0);
50
- const [extracting, setExtracting] = useState(false);
51
- const [preview, setPreview] = useState<string | null>(null);
52
- const [savedName, setSavedName] = useState<string | null>(null);
53
- const [error, setError] = useState<string | null>(null);
54
-
55
- useEffect(() => {
56
- gen.fetchProjectVideos(projectId);
57
- // eslint-disable-next-line react-hooks/exhaustive-deps
58
- }, [projectId]);
59
-
60
- const selectVideo = (v: ProjectVideo) => {
61
- setSelected(v);
62
- setPreview(null);
63
- setSavedName(null);
64
- setDuration(0);
65
- setTime(0);
66
- setError(null);
67
- };
68
-
69
- const onLoadedMetadata = (e: React.SyntheticEvent<HTMLVideoElement>) => {
70
- const d = e.currentTarget.duration;
71
- setDuration(Number.isFinite(d) ? d : 0);
72
- setTime(0);
73
- };
74
-
75
- const handleExtract = async () => {
76
- const video = videoRef.current;
77
- if (!video) return;
78
- setExtracting(true);
79
- setError(null);
80
- setPreview(null);
81
- setSavedName(null);
82
- try {
83
- const dataUrl = await captureFrameAt(video, time);
84
- setPreview(dataUrl);
85
-
86
- const res = await fetch(`${API_BASE}/api/extracted-frames`, {
87
- method: "POST",
88
- headers: { "Content-Type": "application/json" },
89
- body: JSON.stringify({
90
- image: dataUrl,
91
- filename: `frame-${Date.now()}.png`,
92
- projectId,
93
- }),
94
- });
95
- if (!res.ok) throw new Error(await res.text());
96
- const data = await res.json();
97
- setSavedName(data.filename as string);
98
- } catch (e) {
99
- setError(String(e));
100
- } finally {
101
- setExtracting(false);
102
- }
103
- };
104
-
105
- // ========== SVG Icons ==========
106
-
107
- const VideoIcon = (
108
- <svg
109
- width="18"
110
- height="18"
111
- viewBox="0 0 24 24"
112
- fill="none"
113
- stroke="currentColor"
114
- strokeWidth="2"
115
- >
116
- <polygon points="23 7 16 12 23 17 23 7" />
117
- <rect x="1" y="5" width="15" height="14" rx="2" ry="2" />
118
- </svg>
119
- );
120
-
121
- const CameraIcon = (
122
- <svg
123
- width="14"
124
- height="14"
125
- viewBox="0 0 24 24"
126
- fill="none"
127
- stroke="currentColor"
128
- strokeWidth="2"
129
- strokeLinecap="round"
130
- strokeLinejoin="round"
131
- >
132
- <path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" />
133
- <circle cx="12" cy="13" r="4" />
134
- </svg>
135
- );
136
-
137
- return (
138
- <div className="flex flex-col gap-7">
139
- <div className="flex items-center gap-2">
140
- <span className="text-tiffany-600">{CameraIcon}</span>
141
- <h2 className="text-base font-semibold text-ink-900">
142
- Image Extract from Video
143
- </h2>
144
- </div>
145
-
146
- {/* Video picker */}
147
- <div>
148
- <label className="block text-xs font-semibold text-ink-700 uppercase tracking-wider mb-2">
149
- Videos
150
- </label>
151
- {gen.projectVideosLoading ? (
152
- <p className="text-xs text-ink-500 italic py-4 text-center">
153
- Loading videos...
154
- </p>
155
- ) : gen.projectVideos.length === 0 ? (
156
- <p className="text-xs text-ink-500 italic py-4 text-center border border-dashed border-ink-200 rounded-2xl">
157
- No generated videos yet.
158
- </p>
159
- ) : (
160
- <ul className="divide-y divide-ink-200 border border-ink-200 rounded-2xl overflow-hidden max-h-48 overflow-y-auto">
161
- {gen.projectVideos.map((v) => (
162
- <li key={v.filename}>
163
- <button
164
- onClick={() => selectVideo(v)}
165
- className={`w-full flex items-center gap-2 px-3 py-2 text-left text-xs transition-colors ${
166
- selected?.filename === v.filename
167
- ? "bg-ink-100 text-ink-800"
168
- : "text-ink-600 hover:bg-ink-100"
169
- }`}
170
- >
171
- <span className="text-ink-500">{VideoIcon}</span>
172
- <span className="truncate">{v.filename}</span>
173
- </button>
174
- </li>
175
- ))}
176
- </ul>
177
- )}
178
- </div>
179
-
180
- {selected && (
181
- <div className="flex flex-col gap-6">
182
- <video
183
- ref={videoRef}
184
- src={selected.url}
185
- crossOrigin="anonymous"
186
- controls
187
- onLoadedMetadata={onLoadedMetadata}
188
- className="w-full max-h-64 rounded-2xl border border-ink-200 bg-black"
189
- />
190
-
191
- <div>
192
- <label className="block text-xs font-semibold text-ink-700 uppercase tracking-wider mb-2">
193
- Moment ({time.toFixed(2)}s / {duration.toFixed(2)}s)
194
- </label>
195
- <input
196
- type="range"
197
- min={0}
198
- max={duration || 0}
199
- step={0.05}
200
- value={time}
201
- onChange={(e) => {
202
- const t = Number(e.target.value);
203
- setTime(t);
204
- if (videoRef.current) videoRef.current.currentTime = t;
205
- }}
206
- disabled={!duration}
207
- className="w-full accent-tiffany-500"
208
- />
209
- </div>
210
-
211
- <div className="flex items-center gap-2">
212
- <button
213
- onClick={handleExtract}
214
- disabled={extracting || !duration}
215
- className="flex items-center gap-1.5 px-4 py-2 text-xs font-medium rounded-xl bg-tiffany-500 hover:bg-tiffany-600 disabled:bg-ink-200 disabled:text-ink-500 text-ink-950 transition-colors"
216
- >
217
- {CameraIcon}
218
- Extract Frame
219
- </button>
220
- {extracting && (
221
- <span className="text-xs text-ink-600 italic">
222
- Extracting...
223
- </span>
224
- )}
225
- </div>
226
-
227
- {error && <p className="text-xs text-red-600">{error}</p>}
228
-
229
- {preview && (
230
- <div className="flex flex-col gap-3">
231
- <div className="rounded-2xl overflow-hidden border border-ink-200 inline-block bg-ink-100">
232
- <img
233
- src={preview}
234
- alt="extracted"
235
- className="max-h-72 object-contain"
236
- />
237
- </div>
238
- {savedName && (
239
- <p className="text-xs text-emerald-600">
240
- Saved as {savedName} in extracted-frames/{projectId}
241
- </p>
242
- )}
243
- </div>
244
- )}
245
- </div>
246
- )}
247
- </div>
248
- );
249
- }
@@ -1,267 +0,0 @@
1
- import { useEffect, useRef } from "react";
2
- import { useSceneVisualStore } from "../../stores/sceneVisualStore";
3
-
4
- interface Props {
5
- projectId: string;
6
- }
7
-
8
- export default function SceneVisualTab({ projectId }: Props) {
9
- const sceneStore = useSceneVisualStore();
10
-
11
- useEffect(() => {
12
- sceneStore.ensureProject(projectId);
13
- // eslint-disable-next-line react-hooks/exhaustive-deps
14
- }, [projectId]);
15
-
16
- const anyGenerating = sceneStore.items.some((i) => i.generating);
17
-
18
- const uploadFileInputRef = useRef<HTMLInputElement>(null);
19
- const uploadTargetRef = useRef<string | null>(null);
20
-
21
- const handleUploadFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
22
- const file = e.target.files?.[0];
23
- const targetId = uploadTargetRef.current;
24
- e.target.value = "";
25
- if (!file || !targetId) return;
26
- const reader = new FileReader();
27
- reader.onload = () => {
28
- sceneStore.uploadItemImage(
29
- projectId,
30
- targetId,
31
- reader.result as string,
32
- file.name,
33
- );
34
- };
35
- reader.readAsDataURL(file);
36
- };
37
-
38
- // ========== SVG Icons ==========
39
-
40
- const SceneIcon = (
41
- <svg
42
- width="18"
43
- height="18"
44
- viewBox="0 0 24 24"
45
- fill="none"
46
- stroke="currentColor"
47
- strokeWidth="2"
48
- strokeLinecap="round"
49
- strokeLinejoin="round"
50
- >
51
- <rect x="3" y="3" width="18" height="18" rx="2" ry="2" />
52
- <circle cx="8.5" cy="8.5" r="1.5" />
53
- <polyline points="21 15 16 10 5 21" />
54
- </svg>
55
- );
56
-
57
- const PlusIcon = (
58
- <svg
59
- width="16"
60
- height="16"
61
- viewBox="0 0 24 24"
62
- fill="none"
63
- stroke="currentColor"
64
- strokeWidth="2"
65
- strokeLinecap="round"
66
- strokeLinejoin="round"
67
- >
68
- <line x1="12" y1="5" x2="12" y2="19" />
69
- <line x1="5" y1="12" x2="19" y2="12" />
70
- </svg>
71
- );
72
-
73
- const TrashIcon = (
74
- <svg
75
- width="14"
76
- height="14"
77
- viewBox="0 0 24 24"
78
- fill="none"
79
- stroke="currentColor"
80
- strokeWidth="2"
81
- strokeLinecap="round"
82
- strokeLinejoin="round"
83
- >
84
- <polyline points="3 6 5 6 21 6" />
85
- <path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2" />
86
- </svg>
87
- );
88
-
89
- const SparkleIcon = (
90
- <svg
91
- width="14"
92
- height="14"
93
- viewBox="0 0 24 24"
94
- fill="none"
95
- stroke="currentColor"
96
- strokeWidth="2"
97
- >
98
- <polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2" />
99
- </svg>
100
- );
101
-
102
- const SpinnerIcon = (
103
- <svg
104
- className="animate-spin text-tiffany-600"
105
- width="14"
106
- height="14"
107
- viewBox="0 0 24 24"
108
- fill="none"
109
- stroke="currentColor"
110
- strokeWidth="2"
111
- >
112
- <circle cx="12" cy="12" r="10" strokeOpacity="0.25" />
113
- <path d="M12 2a10 10 0 0 1 10 10" strokeOpacity="0.75" />
114
- </svg>
115
- );
116
-
117
- const StopIcon = (
118
- <svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor">
119
- <rect x="4" y="4" width="16" height="16" rx="2" />
120
- </svg>
121
- );
122
-
123
- const UploadIcon = (
124
- <svg
125
- width="14"
126
- height="14"
127
- viewBox="0 0 24 24"
128
- fill="none"
129
- stroke="currentColor"
130
- strokeWidth="2"
131
- strokeLinecap="round"
132
- strokeLinejoin="round"
133
- >
134
- <path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4" />
135
- <polyline points="17 8 12 3 7 8" />
136
- <line x1="12" y1="3" x2="12" y2="15" />
137
- </svg>
138
- );
139
-
140
- return (
141
- <div className="flex flex-col gap-7">
142
- <div className="flex items-center gap-2">
143
- <span className="text-tiffany-600">{SceneIcon}</span>
144
- <h2 className="text-base font-semibold text-ink-900">
145
- Scene Visual
146
- </h2>
147
- </div>
148
-
149
- {/* Halt generation */}
150
- {anyGenerating && (
151
- <button
152
- onClick={() => sceneStore.haltAll()}
153
- className="flex items-center justify-center gap-2 w-full px-4 py-2.5 bg-red-50 hover:bg-red-100 text-red-600 text-sm font-semibold rounded-2xl border border-red-200 transition-colors"
154
- >
155
- {StopIcon}
156
- Stop Generation
157
- </button>
158
- )}
159
-
160
- {/* Add scene */}
161
- <button
162
- onClick={() => sceneStore.addItem()}
163
- className="flex items-center justify-center gap-2 w-full px-4 py-2.5 bg-ink-50 hover:bg-ink-200 text-ink-700 text-sm font-medium rounded-2xl border border-ink-200 transition-colors"
164
- >
165
- {PlusIcon}
166
- Add Scene
167
- </button>
168
-
169
- <input
170
- ref={uploadFileInputRef}
171
- type="file"
172
- accept="image/*"
173
- onChange={handleUploadFileChange}
174
- className="hidden"
175
- />
176
-
177
- {/* Scene items */}
178
- {sceneStore.items.length === 0 ? (
179
- <p className="text-xs text-ink-500 italic text-center py-8 border border-dashed border-ink-200 rounded-2xl">
180
- No scenes yet. Add one to generate a scene visual.
181
- </p>
182
- ) : (
183
- <div className="flex flex-col gap-6">
184
- {sceneStore.items.map((item, index) => (
185
- <div
186
- key={item.id}
187
- className="border border-ink-200 rounded-2xl p-5 flex flex-col gap-3 bg-ink-100/60"
188
- >
189
- <div className="flex items-center justify-between">
190
- <span className="text-xs font-semibold text-ink-700 uppercase tracking-wider">
191
- Scene {index + 1}
192
- </span>
193
- <button
194
- onClick={() => sceneStore.removeItem(item.id)}
195
- disabled={item.generating}
196
- className="flex items-center justify-center w-6 h-6 rounded-full text-ink-500 hover:bg-red-50 hover:text-red-600 disabled:opacity-50 transition-colors"
197
- title="Remove scene"
198
- >
199
- {TrashIcon}
200
- </button>
201
- </div>
202
-
203
- <textarea
204
- value={item.prompt}
205
- onChange={(e) => sceneStore.setPrompt(item.id, e.target.value)}
206
- placeholder="Describe the scene, e.g. a little lamb standing in a sunny meadow."
207
- rows={2}
208
- disabled={item.generating}
209
- className="w-full px-3 py-2 bg-white border border-ink-200 rounded-xl text-ink-900 text-sm placeholder-ink-500 focus:outline-none focus:border-tiffany-500 focus:ring-2 focus:ring-tiffany-500/30 transition-all resize-none disabled:opacity-50"
210
- />
211
-
212
- {item.generating || item.uploading ? (
213
- <div className="flex items-center gap-2 px-3 py-2 bg-ink-50 border border-ink-200 rounded-xl text-xs text-ink-700">
214
- {SpinnerIcon}
215
- {item.generating ? "Generating..." : "Uploading..."}
216
- </div>
217
- ) : (
218
- <div className="flex items-center gap-2">
219
- <button
220
- onClick={() => sceneStore.generateItem(projectId, item.id)}
221
- disabled={!item.prompt.trim()}
222
- className="flex items-center justify-center gap-2 px-4 py-2 text-xs font-medium rounded-xl bg-tiffany-500 hover:bg-tiffany-600 disabled:bg-ink-200 disabled:text-ink-500 text-ink-950 transition-colors"
223
- >
224
- {SparkleIcon}
225
- Generate
226
- </button>
227
- <button
228
- onClick={() => {
229
- uploadTargetRef.current = item.id;
230
- uploadFileInputRef.current?.click();
231
- }}
232
- className="flex items-center justify-center gap-2 px-4 py-2 text-xs font-medium rounded-xl border border-ink-200 bg-white text-ink-600 hover:border-ink-300 transition-colors"
233
- >
234
- {UploadIcon}
235
- Upload Image
236
- </button>
237
- </div>
238
- )}
239
-
240
- {item.error && (
241
- <p className="text-xs text-red-600">{item.error}</p>
242
- )}
243
-
244
- {item.logs.length > 0 && (
245
- <div className="p-2 bg-ink-50 border border-ink-200 rounded-xl max-h-24 overflow-y-auto">
246
- <pre className="text-[10px] text-ink-600 font-mono whitespace-pre-wrap">
247
- {item.logs.join("")}
248
- </pre>
249
- </div>
250
- )}
251
-
252
- {item.result && (
253
- <div className="rounded-xl overflow-hidden border border-ink-200 inline-block">
254
- <img
255
- src={item.result}
256
- alt={`Scene ${index + 1}`}
257
- className="max-w-full h-auto"
258
- />
259
- </div>
260
- )}
261
- </div>
262
- ))}
263
- </div>
264
- )}
265
- </div>
266
- );
267
- }
@@ -1,75 +0,0 @@
1
- import localforage from "localforage";
2
- import type {
3
- AspectRatio,
4
- Resolution,
5
- VideoMode,
6
- } from "../stores/generationStore";
7
-
8
- // TTS quality tier: "high" = Qwen3 1.7B, "low" = Qwen3 0.6B.
9
- export type VoiceQuality = "low" | "high";
10
-
11
- // Persisted "UI state" for the batch custom voice video tab: the editable
12
- // setup (rows + shared video settings + voice reference). Transient
13
- // generation state (status, results, logs) is intentionally not persisted.
14
- export interface PersistedBatchVoiceRow {
15
- id: string;
16
- prompt: string;
17
- script: string;
18
- imagePath: string | null;
19
- imageUrl: string | null;
20
- imageFilename: string | null;
21
- }
22
-
23
- export interface PersistedBatchVoiceState {
24
- rows: PersistedBatchVoiceRow[];
25
- duration: number;
26
- aspectRatio: AspectRatio;
27
- resolution: Resolution;
28
- mode: VideoMode;
29
- quality: VoiceQuality;
30
- voiceRefPath: string | null;
31
- voiceRefFilename: string | null;
32
- }
33
-
34
- const store = localforage.createInstance({
35
- name: "lambobo-studio",
36
- storeName: "batch-voice",
37
- });
38
-
39
- // Each project keeps its own persisted batch-voice UI state, keyed by
40
- // projectId so switching projects restores the right rows + settings.
41
- function storageKey(projectId: string): string {
42
- return `batch-voice-ui-state:${projectId}`;
43
- }
44
-
45
- export async function loadBatchVoiceState(
46
- projectId: string,
47
- ): Promise<PersistedBatchVoiceState | null> {
48
- try {
49
- const value = await store.getItem<PersistedBatchVoiceState>(
50
- storageKey(projectId),
51
- );
52
- return value ?? null;
53
- } catch {
54
- return null;
55
- }
56
- }
57
-
58
- export async function saveBatchVoiceState(
59
- projectId: string,
60
- state: PersistedBatchVoiceState,
61
- ): Promise<void> {
62
- try {
63
- await store.setItem(storageKey(projectId), state);
64
- } catch {
65
- // Ignore persistence failures — the UI keeps working in memory.
66
- }
67
- }
68
-
69
- export async function clearBatchVoiceState(projectId: string): Promise<void> {
70
- try {
71
- await store.removeItem(storageKey(projectId));
72
- } catch {
73
- // Ignore — nothing to clear.
74
- }
75
- }