mulmocast 2.3.2 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/html/tailwind.html +1 -0
- package/assets/html/tailwind_animated.html +259 -0
- package/lib/actions/image_agents.d.ts +1 -0
- package/lib/actions/image_agents.js +24 -2
- package/lib/actions/images.d.ts +3 -0
- package/lib/actions/images.js +4 -3
- package/lib/agents/combine_audio_files_agent.js +5 -0
- package/lib/methods/mulmo_beat.d.ts +7 -0
- package/lib/methods/mulmo_beat.js +18 -0
- package/lib/types/schema.d.ts +23 -0
- package/lib/types/schema.js +10 -0
- package/lib/utils/context.d.ts +8 -0
- package/lib/utils/ffmpeg_utils.d.ts +11 -1
- package/lib/utils/ffmpeg_utils.js +33 -2
- package/lib/utils/file.d.ts +1 -0
- package/lib/utils/file.js +6 -0
- package/lib/utils/html_render.d.ts +11 -0
- package/lib/utils/html_render.js +84 -34
- package/lib/utils/image_plugins/html_tailwind.js +78 -6
- package/package.json +3 -3
- package/scripts/test/test_html_animation.json +563 -0
- package/scripts/test/test_vocab_animation.json +226 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en" class="h-full">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
|
+
<script src="https://cdn.tailwindcss.com"></script>
|
|
7
|
+
<style>
|
|
8
|
+
/* Disable all CSS animations/transitions for deterministic frame-based rendering */
|
|
9
|
+
*, *::before, *::after {
|
|
10
|
+
animation-play-state: paused !important;
|
|
11
|
+
transition: none !important;
|
|
12
|
+
}
|
|
13
|
+
${custom_style}
|
|
14
|
+
</style>
|
|
15
|
+
</head>
|
|
16
|
+
<body class="bg-white text-gray-800 h-full flex flex-col">
|
|
17
|
+
${html_body}
|
|
18
|
+
|
|
19
|
+
<script>
|
|
20
|
+
// === MulmoCast Animation Helpers ===
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Easing functions for non-linear interpolation.
|
|
24
|
+
*/
|
|
25
|
+
const Easing = {
|
|
26
|
+
linear: (t) => t,
|
|
27
|
+
easeIn: (t) => t * t,
|
|
28
|
+
easeOut: (t) => 1 - (1 - t) * (1 - t),
|
|
29
|
+
easeInOut: (t) => t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2,
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Interpolation with clamping and optional easing.
|
|
34
|
+
*
|
|
35
|
+
* @param {number} value - Current value (typically frame number)
|
|
36
|
+
* @param {Object} opts - { input: { inMin, inMax }, output: { outMin, outMax }, easing?: string | function }
|
|
37
|
+
* @returns {number} Interpolated and clamped value
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* interpolate(frame, { input: { inMin: 0, inMax: 30 }, output: { outMin: 0, outMax: 1 } })
|
|
41
|
+
* interpolate(frame, { input: { inMin: 0, inMax: 30 }, output: { outMin: 0, outMax: 1 }, easing: 'easeOut' })
|
|
42
|
+
*/
|
|
43
|
+
function interpolate(value, opts) {
|
|
44
|
+
const { inMin, inMax } = opts.input;
|
|
45
|
+
const { outMin, outMax } = opts.output;
|
|
46
|
+
if (inMax === inMin) {
|
|
47
|
+
return outMin;
|
|
48
|
+
}
|
|
49
|
+
const easing = !opts.easing ? Easing.linear
|
|
50
|
+
: typeof opts.easing === 'function' ? opts.easing
|
|
51
|
+
: Easing[opts.easing] || Easing.linear;
|
|
52
|
+
const progress = Math.max(0, Math.min(1, (value - inMin) / (inMax - inMin)));
|
|
53
|
+
return outMin + easing(progress) * (outMax - outMin);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// === MulmoAnimation Helper Class ===
|
|
57
|
+
|
|
58
|
+
const TRANSFORM_PROPS = { translateX: 'px', translateY: 'px', scale: '', rotate: 'deg' };
|
|
59
|
+
const SVG_PROPS = ['r', 'cx', 'cy', 'x', 'y', 'x1', 'y1', 'x2', 'y2', 'rx', 'ry',
|
|
60
|
+
'width', 'height', 'stroke-width', 'stroke-dashoffset', 'stroke-dasharray', 'opacity'];
|
|
61
|
+
|
|
62
|
+
function MulmoAnimation() {
|
|
63
|
+
this._entries = [];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Register a property animation on a single element.
|
|
68
|
+
* @param {string} selector - CSS selector (e.g. '#title')
|
|
69
|
+
* @param {Object} props - { opacity: [0, 1], translateY: [30, 0], width: [0, 80, '%'] }
|
|
70
|
+
* @param {Object} opts - { start, end, easing } (start/end in seconds)
|
|
71
|
+
*/
|
|
72
|
+
MulmoAnimation.prototype.animate = function(selector, props, opts) {
|
|
73
|
+
this._entries.push({ kind: 'animate', selector, props, opts: opts || {} });
|
|
74
|
+
return this;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Stagger animation across numbered elements.
|
|
79
|
+
* Selector must contain {i} placeholder (e.g. '#item{i}').
|
|
80
|
+
* @param {string} selector - e.g. '#item{i}'
|
|
81
|
+
* @param {number} count - number of elements (0-indexed)
|
|
82
|
+
* @param {Object} props - same as animate()
|
|
83
|
+
* @param {Object} opts - { start, stagger, duration, easing }
|
|
84
|
+
*/
|
|
85
|
+
MulmoAnimation.prototype.stagger = function(selector, count, props, opts) {
|
|
86
|
+
this._entries.push({ kind: 'stagger', selector, count, props, opts: opts || {} });
|
|
87
|
+
return this;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Typewriter effect — reveal text character by character.
|
|
92
|
+
* @param {string} selector - target element selector
|
|
93
|
+
* @param {string} text - full text to reveal
|
|
94
|
+
* @param {Object} opts - { start, end }
|
|
95
|
+
*/
|
|
96
|
+
MulmoAnimation.prototype.typewriter = function(selector, text, opts) {
|
|
97
|
+
this._entries.push({ kind: 'typewriter', selector, text, opts: opts || {} });
|
|
98
|
+
return this;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Animated counter — interpolate a number and display with optional prefix/suffix.
|
|
103
|
+
* @param {string} selector - target element selector
|
|
104
|
+
* @param {[number, number]} range - [from, to]
|
|
105
|
+
* @param {Object} opts - { start, end, prefix, suffix, decimals }
|
|
106
|
+
*/
|
|
107
|
+
MulmoAnimation.prototype.counter = function(selector, range, opts) {
|
|
108
|
+
this._entries.push({ kind: 'counter', selector, range, opts: opts || {} });
|
|
109
|
+
return this;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Code reveal — show lines of code one by one (line-level typewriter).
|
|
114
|
+
* @param {string} selector - target element selector
|
|
115
|
+
* @param {string[]} lines - array of code lines
|
|
116
|
+
* @param {Object} opts - { start, end }
|
|
117
|
+
*/
|
|
118
|
+
MulmoAnimation.prototype.codeReveal = function(selector, lines, opts) {
|
|
119
|
+
this._entries.push({ kind: 'codeReveal', selector, lines, opts: opts || {} });
|
|
120
|
+
return this;
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Blink — periodic show/hide toggle (e.g. cursor blinking).
|
|
125
|
+
* @param {string} selector - target element selector
|
|
126
|
+
* @param {Object} opts - { interval } (half-cycle seconds, default 0.5)
|
|
127
|
+
*/
|
|
128
|
+
MulmoAnimation.prototype.blink = function(selector, opts) {
|
|
129
|
+
this._entries.push({ kind: 'blink', selector, opts: opts || {} });
|
|
130
|
+
return this;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
/** Resolve easing name string or function to an easing function */
|
|
134
|
+
MulmoAnimation.prototype._resolveEasing = function(e) {
|
|
135
|
+
if (!e) return Easing.linear;
|
|
136
|
+
if (typeof e === 'function') return e;
|
|
137
|
+
return Easing[e] || Easing.linear;
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
/** Apply props to element at a given progress (0-1) with easing */
|
|
141
|
+
MulmoAnimation.prototype._applyProps = function(el, props, progress, easingFn) {
|
|
142
|
+
if (!el) return;
|
|
143
|
+
const transforms = [];
|
|
144
|
+
Object.keys(props).forEach((prop) => {
|
|
145
|
+
const spec = props[prop];
|
|
146
|
+
const from = spec[0], to = spec[1];
|
|
147
|
+
const unit = (spec.length > 2) ? spec[2] : null;
|
|
148
|
+
const val = from + easingFn(progress) * (to - from);
|
|
149
|
+
|
|
150
|
+
if (TRANSFORM_PROPS.hasOwnProperty(prop)) {
|
|
151
|
+
const tUnit = unit || TRANSFORM_PROPS[prop];
|
|
152
|
+
transforms.push(prop === 'scale' ? 'scale(' + val + ')' : prop + '(' + val + tUnit + ')');
|
|
153
|
+
} else if (el instanceof SVGElement && SVG_PROPS.indexOf(prop) !== -1) {
|
|
154
|
+
el.setAttribute(prop, val);
|
|
155
|
+
} else if (prop === 'opacity') {
|
|
156
|
+
el.style.opacity = val;
|
|
157
|
+
} else {
|
|
158
|
+
const cssUnit = unit || 'px';
|
|
159
|
+
el.style[prop] = val + cssUnit;
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
if (transforms.length > 0) {
|
|
163
|
+
el.style.transform = transforms.join(' ');
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Update all registered animations for the given frame.
|
|
169
|
+
* @param {number} frame - current frame number
|
|
170
|
+
* @param {number} fps - frames per second
|
|
171
|
+
*/
|
|
172
|
+
MulmoAnimation.prototype.update = function(frame, fps) {
|
|
173
|
+
this._entries.forEach((entry) => {
|
|
174
|
+
const opts = entry.opts;
|
|
175
|
+
const easingFn = this._resolveEasing(opts.easing);
|
|
176
|
+
|
|
177
|
+
if (entry.kind === 'animate') {
|
|
178
|
+
const startFrame = (opts.start || 0) * fps;
|
|
179
|
+
const endFrame = (opts.end || 0) * fps;
|
|
180
|
+
const progress = Math.max(0, Math.min(1, endFrame === startFrame ? 1 : (frame - startFrame) / (endFrame - startFrame)));
|
|
181
|
+
const el = document.querySelector(entry.selector);
|
|
182
|
+
this._applyProps(el, entry.props, progress, easingFn);
|
|
183
|
+
|
|
184
|
+
} else if (entry.kind === 'stagger') {
|
|
185
|
+
const baseStart = (opts.start || 0) * fps;
|
|
186
|
+
const staggerDelay = (opts.stagger || 0.2) * fps;
|
|
187
|
+
const dur = (opts.duration || 0.5) * fps;
|
|
188
|
+
for (let j = 0; j < entry.count; j++) {
|
|
189
|
+
const sel = entry.selector.replace(/\{i\}/g, j);
|
|
190
|
+
const sEl = document.querySelector(sel);
|
|
191
|
+
const sStart = baseStart + j * staggerDelay;
|
|
192
|
+
const sEnd = sStart + dur;
|
|
193
|
+
const sProgress = Math.max(0, Math.min(1, sEnd === sStart ? 1 : (frame - sStart) / (sEnd - sStart)));
|
|
194
|
+
this._applyProps(sEl, entry.props, sProgress, easingFn);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
} else if (entry.kind === 'typewriter') {
|
|
198
|
+
const twStart = (opts.start || 0) * fps;
|
|
199
|
+
const twEnd = (opts.end || 0) * fps;
|
|
200
|
+
const twProgress = Math.max(0, Math.min(1, twEnd === twStart ? 1 : (frame - twStart) / (twEnd - twStart)));
|
|
201
|
+
const charCount = Math.floor(twProgress * entry.text.length);
|
|
202
|
+
const twEl = document.querySelector(entry.selector);
|
|
203
|
+
if (twEl) twEl.textContent = entry.text.substring(0, charCount);
|
|
204
|
+
|
|
205
|
+
} else if (entry.kind === 'counter') {
|
|
206
|
+
const cStart = (opts.start || 0) * fps;
|
|
207
|
+
const cEnd = (opts.end || 0) * fps;
|
|
208
|
+
const cProgress = Math.max(0, Math.min(1, cEnd === cStart ? 1 : (frame - cStart) / (cEnd - cStart)));
|
|
209
|
+
const cVal = entry.range[0] + easingFn(cProgress) * (entry.range[1] - entry.range[0]);
|
|
210
|
+
const decimals = opts.decimals || 0;
|
|
211
|
+
const display = (opts.prefix || '') + cVal.toFixed(decimals) + (opts.suffix || '');
|
|
212
|
+
const cEl = document.querySelector(entry.selector);
|
|
213
|
+
if (cEl) cEl.textContent = display;
|
|
214
|
+
|
|
215
|
+
} else if (entry.kind === 'codeReveal') {
|
|
216
|
+
const crStart = (opts.start || 0) * fps;
|
|
217
|
+
const crEnd = (opts.end || 0) * fps;
|
|
218
|
+
const crProgress = Math.max(0, Math.min(1, crEnd === crStart ? 1 : (frame - crStart) / (crEnd - crStart)));
|
|
219
|
+
const lineCount = Math.floor(crProgress * entry.lines.length);
|
|
220
|
+
const crEl = document.querySelector(entry.selector);
|
|
221
|
+
if (crEl) crEl.textContent = entry.lines.slice(0, lineCount).join('\n');
|
|
222
|
+
|
|
223
|
+
} else if (entry.kind === 'blink') {
|
|
224
|
+
const interval_s = opts.interval || 0.5;
|
|
225
|
+
const blinkEl = document.querySelector(entry.selector);
|
|
226
|
+
if (blinkEl) {
|
|
227
|
+
const cycle = (frame / fps) / interval_s;
|
|
228
|
+
blinkEl.style.opacity = (Math.floor(cycle) % 2 === 0) ? 1 : 0;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
// === MulmoCast Frame State (updated by Puppeteer per frame) ===
|
|
235
|
+
window.__MULMO = {
|
|
236
|
+
frame: 0,
|
|
237
|
+
totalFrames: ${totalFrames},
|
|
238
|
+
fps: ${fps},
|
|
239
|
+
};
|
|
240
|
+
</script>
|
|
241
|
+
|
|
242
|
+
${user_script}
|
|
243
|
+
|
|
244
|
+
<script>
|
|
245
|
+
// Auto-render: if MulmoAnimation is used but render() is not defined, generate it
|
|
246
|
+
if (typeof render !== 'function' && typeof animation !== 'undefined' && animation instanceof MulmoAnimation) {
|
|
247
|
+
window.render = function(frame, totalFrames, fps) { animation.update(frame, fps); };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Initial render (frame 0)
|
|
251
|
+
if (typeof render === 'function') {
|
|
252
|
+
const result = render(0, window.__MULMO.totalFrames, window.__MULMO.fps);
|
|
253
|
+
if (result && typeof result.then === 'function') {
|
|
254
|
+
result.catch(console.error);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
</script>
|
|
258
|
+
</body>
|
|
259
|
+
</html>
|
|
@@ -41,6 +41,7 @@ type ImageHtmlPreprocessAgentResponse = {
|
|
|
41
41
|
};
|
|
42
42
|
type ImageOnlyMoviePreprocessAgentResponse = ImagePreprocessAgentResponseBase & {
|
|
43
43
|
imageFromMovie: boolean;
|
|
44
|
+
useLastFrame?: boolean;
|
|
44
45
|
};
|
|
45
46
|
type ImagePluginPreprocessAgentResponse = ImagePreprocessAgentResponseBase & {
|
|
46
47
|
referenceImageForMovie: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
|
|
3
|
-
import { getBeatPngImagePath, getBeatMoviePaths, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
|
|
3
|
+
import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
|
|
4
4
|
import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
|
|
5
5
|
import { renderHTMLToImage } from "../utils/html_render.js";
|
|
6
6
|
import { beatId } from "../utils/utils.js";
|
|
@@ -72,6 +72,25 @@ export const imagePreprocessAgent = async (namedInputs) => {
|
|
|
72
72
|
const markdown = plugin.markdown ? plugin.markdown({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
|
|
73
73
|
const html = plugin.html ? await plugin.html({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
|
|
74
74
|
const isTypeMovie = beat.image.type === "movie";
|
|
75
|
+
const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
|
|
76
|
+
// animation and moviePrompt cannot be used together
|
|
77
|
+
if (isAnimatedHtml && beat.moviePrompt) {
|
|
78
|
+
throw new Error("html_tailwind animation and moviePrompt cannot be used together on the same beat. Use either animation or moviePrompt, not both.");
|
|
79
|
+
}
|
|
80
|
+
if (isAnimatedHtml) {
|
|
81
|
+
const animatedVideoPath = getBeatAnimatedVideoPath(context, index);
|
|
82
|
+
// ImagePluginPreprocessAgentResponse
|
|
83
|
+
return {
|
|
84
|
+
...returnValue,
|
|
85
|
+
imagePath, // for thumbnail extraction
|
|
86
|
+
movieFile: animatedVideoPath, // .mp4 path for the pipeline
|
|
87
|
+
imageFromMovie: true, // triggers extractImageFromMovie
|
|
88
|
+
useLastFrame: true, // extract last frame for PDF/static (animation complete state)
|
|
89
|
+
referenceImageForMovie: pluginPath,
|
|
90
|
+
markdown,
|
|
91
|
+
html,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
75
94
|
// undefined prompt indicates that image generation is not needed
|
|
76
95
|
// ImagePluginPreprocessAgentResponse
|
|
77
96
|
return {
|
|
@@ -99,9 +118,12 @@ export const imagePluginAgent = async (namedInputs) => {
|
|
|
99
118
|
const { context, beat, index, imageRefs } = namedInputs;
|
|
100
119
|
const { imagePath } = getBeatPngImagePath(context, index);
|
|
101
120
|
const plugin = MulmoBeatMethods.getPlugin(beat);
|
|
121
|
+
// For animated html_tailwind, use the .mp4 path so the plugin writes video there
|
|
122
|
+
const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
|
|
123
|
+
const effectiveImagePath = isAnimatedHtml ? getBeatAnimatedVideoPath(context, index) : imagePath;
|
|
102
124
|
try {
|
|
103
125
|
MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);
|
|
104
|
-
const processorParams = { beat, context, imagePath, imageRefs, ...htmlStyle(context, beat) };
|
|
126
|
+
const processorParams = { beat, context, imagePath: effectiveImagePath, imageRefs, ...htmlStyle(context, beat) };
|
|
105
127
|
await plugin.process(processorParams);
|
|
106
128
|
MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, false);
|
|
107
129
|
}
|
package/lib/actions/images.d.ts
CHANGED
|
@@ -118,6 +118,7 @@ export declare const beat_graph_data: {
|
|
|
118
118
|
imagePath?: string;
|
|
119
119
|
} & {
|
|
120
120
|
imageFromMovie: boolean;
|
|
121
|
+
useLastFrame?: boolean;
|
|
121
122
|
}) | ({
|
|
122
123
|
imageParams?: MulmoImageParams;
|
|
123
124
|
movieFile?: string;
|
|
@@ -286,11 +287,13 @@ export declare const beat_graph_data: {
|
|
|
286
287
|
agent: (namedInputs: {
|
|
287
288
|
movieFile: string;
|
|
288
289
|
imageFile: string;
|
|
290
|
+
useLastFrame: boolean;
|
|
289
291
|
}) => Promise<object>;
|
|
290
292
|
inputs: {
|
|
291
293
|
onComplete: string[];
|
|
292
294
|
imageFile: string;
|
|
293
295
|
movieFile: string;
|
|
296
|
+
useLastFrame: string;
|
|
294
297
|
};
|
|
295
298
|
defaultValue: {};
|
|
296
299
|
};
|
package/lib/actions/images.js
CHANGED
|
@@ -187,12 +187,13 @@ export const beat_graph_data = {
|
|
|
187
187
|
imageFromMovie: {
|
|
188
188
|
if: ":preprocessor.imageFromMovie",
|
|
189
189
|
agent: async (namedInputs) => {
|
|
190
|
-
return await extractImageFromMovie(namedInputs.movieFile, namedInputs.imageFile);
|
|
190
|
+
return await extractImageFromMovie(namedInputs.movieFile, namedInputs.imageFile, namedInputs.useLastFrame);
|
|
191
191
|
},
|
|
192
192
|
inputs: {
|
|
193
|
-
onComplete: [":movieGenerator"], //
|
|
193
|
+
onComplete: [":movieGenerator", ":imagePlugin"], // :imagePlugin for animated html_tailwind video generation
|
|
194
194
|
imageFile: ":preprocessor.imagePath",
|
|
195
195
|
movieFile: ":preprocessor.movieFile",
|
|
196
|
+
useLastFrame: ":preprocessor.useLastFrame",
|
|
196
197
|
},
|
|
197
198
|
defaultValue: {},
|
|
198
199
|
},
|
|
@@ -219,7 +220,7 @@ export const beat_graph_data = {
|
|
|
219
220
|
}
|
|
220
221
|
},
|
|
221
222
|
inputs: {
|
|
222
|
-
onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"],
|
|
223
|
+
onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator", ":imagePlugin"], // :imagePlugin for animated html_tailwind video generation
|
|
223
224
|
movieFile: ":preprocessor.movieFile",
|
|
224
225
|
imageFile: ":preprocessor.imagePath",
|
|
225
226
|
soundEffectFile: ":preprocessor.soundEffectFile",
|
|
@@ -2,6 +2,7 @@ import { assert, GraphAILogger } from "graphai";
|
|
|
2
2
|
import { silent60secPath, isFile } from "../utils/file.js";
|
|
3
3
|
import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
|
|
4
4
|
import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
|
|
5
|
+
import { MulmoBeatMethods } from "../methods/index.js";
|
|
5
6
|
import { userAssert } from "../utils/utils.js";
|
|
6
7
|
import { getAudioInputIdsError } from "../utils/error_cause.js";
|
|
7
8
|
const getMovieDuration = async (context, beat) => {
|
|
@@ -13,6 +14,10 @@ const getMovieDuration = async (context, beat) => {
|
|
|
13
14
|
return { duration: duration / speed, hasAudio };
|
|
14
15
|
}
|
|
15
16
|
}
|
|
17
|
+
// Animated html_tailwind beats with explicit duration act as movie-like for voice_over grouping
|
|
18
|
+
if (MulmoBeatMethods.isAnimatedHtmlTailwind(beat) && beat.duration !== undefined) {
|
|
19
|
+
return { duration: beat.duration, hasAudio: false };
|
|
20
|
+
}
|
|
16
21
|
return { duration: 0, hasAudio: false };
|
|
17
22
|
};
|
|
18
23
|
export const getPadding = (context, beat, index) => {
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { MulmoBeat } from "../types/index.js";
|
|
2
|
+
type AnimationConfig = {
|
|
3
|
+
fps?: number;
|
|
4
|
+
};
|
|
2
5
|
export declare const MulmoBeatMethods: {
|
|
6
|
+
isAnimationEnabled: (animation: unknown) => animation is true | AnimationConfig;
|
|
7
|
+
isAnimationObject: (animation: unknown) => animation is AnimationConfig;
|
|
8
|
+
isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
|
|
3
9
|
getHtmlPrompt(beat: MulmoBeat): string | undefined;
|
|
4
10
|
getPlugin(beat: MulmoBeat): {
|
|
5
11
|
imageType: string;
|
|
@@ -10,3 +16,4 @@ export declare const MulmoBeatMethods: {
|
|
|
10
16
|
};
|
|
11
17
|
getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
|
|
12
18
|
};
|
|
19
|
+
export {};
|
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
import { findImagePlugin } from "../utils/image_plugins/index.js";
|
|
2
|
+
/** Type guard: checks if animation value is an object config like { fps: 30 } */
|
|
3
|
+
const isAnimationObject = (animation) => {
|
|
4
|
+
return typeof animation === "object" && animation !== null && !Array.isArray(animation);
|
|
5
|
+
};
|
|
6
|
+
/** Check if a value is a valid animation config (true or non-array object) */
|
|
7
|
+
const isAnimationEnabled = (animation) => {
|
|
8
|
+
return animation === true || isAnimationObject(animation);
|
|
9
|
+
};
|
|
10
|
+
/** Check if a beat has html_tailwind animation enabled */
|
|
11
|
+
const isAnimatedHtmlTailwind = (beat) => {
|
|
12
|
+
if (!beat.image || beat.image.type !== "html_tailwind")
|
|
13
|
+
return false;
|
|
14
|
+
const animation = beat.image.animation;
|
|
15
|
+
return isAnimationEnabled(animation);
|
|
16
|
+
};
|
|
2
17
|
export const MulmoBeatMethods = {
|
|
18
|
+
isAnimationEnabled,
|
|
19
|
+
isAnimationObject,
|
|
20
|
+
isAnimatedHtmlTailwind,
|
|
3
21
|
getHtmlPrompt(beat) {
|
|
4
22
|
if (beat?.htmlPrompt?.data) {
|
|
5
23
|
return beat.htmlPrompt.prompt + "\n\n[data]\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);
|
package/lib/types/schema.d.ts
CHANGED
|
@@ -371,9 +371,16 @@ export declare const mulmoMermaidMediaSchema: z.ZodObject<{
|
|
|
371
371
|
opacity: z.ZodOptional<z.ZodNumber>;
|
|
372
372
|
}, z.core.$strip>]>>>;
|
|
373
373
|
}, z.core.$strict>;
|
|
374
|
+
export declare const htmlTailwindAnimationSchema: z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
|
|
375
|
+
fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
376
|
+
}, z.core.$strip>]>;
|
|
374
377
|
export declare const mulmoHtmlTailwindMediaSchema: z.ZodObject<{
|
|
375
378
|
type: z.ZodLiteral<"html_tailwind">;
|
|
376
379
|
html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
|
|
380
|
+
script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
|
|
381
|
+
animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
|
|
382
|
+
fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
383
|
+
}, z.core.$strip>]>>;
|
|
377
384
|
}, z.core.$strict>;
|
|
378
385
|
export declare const mulmoBeatReferenceMediaSchema: z.ZodObject<{
|
|
379
386
|
type: z.ZodLiteral<"beat">;
|
|
@@ -542,6 +549,10 @@ export declare const mulmoImageAssetSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
542
549
|
}, z.core.$strict>, z.ZodObject<{
|
|
543
550
|
type: z.ZodLiteral<"html_tailwind">;
|
|
544
551
|
html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
|
|
552
|
+
script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
|
|
553
|
+
animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
|
|
554
|
+
fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
555
|
+
}, z.core.$strip>]>>;
|
|
545
556
|
}, z.core.$strict>, z.ZodObject<{
|
|
546
557
|
type: z.ZodLiteral<"beat">;
|
|
547
558
|
id: z.ZodOptional<z.ZodString>;
|
|
@@ -3591,6 +3602,10 @@ export declare const mulmoBeatSchema: z.ZodObject<{
|
|
|
3591
3602
|
}, z.core.$strict>, z.ZodObject<{
|
|
3592
3603
|
type: z.ZodLiteral<"html_tailwind">;
|
|
3593
3604
|
html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
|
|
3605
|
+
script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
|
|
3606
|
+
animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
|
|
3607
|
+
fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
3608
|
+
}, z.core.$strip>]>>;
|
|
3594
3609
|
}, z.core.$strict>, z.ZodObject<{
|
|
3595
3610
|
type: z.ZodLiteral<"beat">;
|
|
3596
3611
|
id: z.ZodOptional<z.ZodString>;
|
|
@@ -7382,6 +7397,10 @@ export declare const mulmoScriptSchema: z.ZodObject<{
|
|
|
7382
7397
|
}, z.core.$strict>, z.ZodObject<{
|
|
7383
7398
|
type: z.ZodLiteral<"html_tailwind">;
|
|
7384
7399
|
html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
|
|
7400
|
+
script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
|
|
7401
|
+
animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
|
|
7402
|
+
fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
7403
|
+
}, z.core.$strip>]>>;
|
|
7385
7404
|
}, z.core.$strict>, z.ZodObject<{
|
|
7386
7405
|
type: z.ZodLiteral<"beat">;
|
|
7387
7406
|
id: z.ZodOptional<z.ZodString>;
|
|
@@ -10793,6 +10812,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
10793
10812
|
}, z.core.$strict>, z.ZodObject<{
|
|
10794
10813
|
type: z.ZodLiteral<"html_tailwind">;
|
|
10795
10814
|
html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
|
|
10815
|
+
script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
|
|
10816
|
+
animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
|
|
10817
|
+
fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
10818
|
+
}, z.core.$strip>]>>;
|
|
10796
10819
|
}, z.core.$strict>, z.ZodObject<{
|
|
10797
10820
|
type: z.ZodLiteral<"beat">;
|
|
10798
10821
|
id: z.ZodOptional<z.ZodString>;
|
package/lib/types/schema.js
CHANGED
|
@@ -193,10 +193,20 @@ export const mulmoMermaidMediaSchema = z
|
|
|
193
193
|
backgroundImage: backgroundImageSchema,
|
|
194
194
|
})
|
|
195
195
|
.strict();
|
|
196
|
+
export const htmlTailwindAnimationSchema = z.union([
|
|
197
|
+
z.literal(true),
|
|
198
|
+
z.object({
|
|
199
|
+
fps: z.number().min(1).max(60).optional().default(30),
|
|
200
|
+
}),
|
|
201
|
+
]);
|
|
196
202
|
export const mulmoHtmlTailwindMediaSchema = z
|
|
197
203
|
.object({
|
|
198
204
|
type: z.literal("html_tailwind"),
|
|
199
205
|
html: stringOrStringArray,
|
|
206
|
+
script: stringOrStringArray.optional().describe("JavaScript code for the beat. Injected as a <script> tag after html. Use for render() function etc."),
|
|
207
|
+
animation: htmlTailwindAnimationSchema
|
|
208
|
+
.optional()
|
|
209
|
+
.describe("Enable frame-based animation (Remotion-style). true for defaults (30fps), or { fps: N } for custom frame rate."),
|
|
200
210
|
})
|
|
201
211
|
.strict();
|
|
202
212
|
export const mulmoBeatReferenceMediaSchema = z
|
package/lib/utils/context.d.ts
CHANGED
|
@@ -1722,6 +1722,10 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
1722
1722
|
} | {
|
|
1723
1723
|
type: "html_tailwind";
|
|
1724
1724
|
html: string | string[];
|
|
1725
|
+
script?: string | string[] | undefined;
|
|
1726
|
+
animation?: true | {
|
|
1727
|
+
fps: number;
|
|
1728
|
+
} | undefined;
|
|
1725
1729
|
} | {
|
|
1726
1730
|
type: "beat";
|
|
1727
1731
|
id?: string | undefined;
|
|
@@ -3810,6 +3814,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
3810
3814
|
} | {
|
|
3811
3815
|
type: "html_tailwind";
|
|
3812
3816
|
html: string | string[];
|
|
3817
|
+
script?: string | string[] | undefined;
|
|
3818
|
+
animation?: true | {
|
|
3819
|
+
fps: number;
|
|
3820
|
+
} | undefined;
|
|
3813
3821
|
} | {
|
|
3814
3822
|
type: "beat";
|
|
3815
3823
|
id?: string | undefined;
|
|
@@ -11,11 +11,21 @@ export declare const FfmpegContextAddInput: (context: FfmpegContext, input: stri
|
|
|
11
11
|
export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
|
|
12
12
|
export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined, inputOptions?: string[]) => string;
|
|
13
13
|
export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;
|
|
14
|
+
/** Round up odd dimensions to even (required by libx264 yuv420p) */
|
|
15
|
+
export declare const normalizeEvenDimensions: (width: number, height: number) => {
|
|
16
|
+
width: number;
|
|
17
|
+
height: number;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Convert a sequence of PNG frames into a video file.
|
|
21
|
+
* Expects files named frame_00000.png, frame_00001.png, etc. in framesDir.
|
|
22
|
+
*/
|
|
23
|
+
export declare const framesToVideo: (framesDir: string, outputPath: string, fps: number, width: number, height: number) => Promise<void>;
|
|
14
24
|
export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
|
|
15
25
|
duration: number;
|
|
16
26
|
hasAudio: boolean;
|
|
17
27
|
}>;
|
|
18
|
-
export declare const extractImageFromMovie: (movieFile: string, imagePath: string) => Promise<object>;
|
|
28
|
+
export declare const extractImageFromMovie: (movieFile: string, imagePath: string, useLastFrame?: boolean) => Promise<object>;
|
|
19
29
|
export declare const trimMusic: (inputFile: string, startTime: number, duration: number) => Promise<Buffer>;
|
|
20
30
|
export declare const createSilentAudio: (filePath: string, durationSec: number) => Promise<void>;
|
|
21
31
|
export declare const pcmToMp3: (rawPcm: Buffer, sampleRate?: number) => Promise<Buffer>;
|
|
@@ -62,6 +62,33 @@ export const FfmpegContextGenerateOutput = (context, output, options = []) => {
|
|
|
62
62
|
.run();
|
|
63
63
|
});
|
|
64
64
|
};
|
|
65
|
+
/** Round up odd dimensions to even (required by libx264 yuv420p) */
|
|
66
|
+
export const normalizeEvenDimensions = (width, height) => {
|
|
67
|
+
return {
|
|
68
|
+
width: width % 2 === 0 ? width : width + 1,
|
|
69
|
+
height: height % 2 === 0 ? height : height + 1,
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
/**
|
|
73
|
+
* Convert a sequence of PNG frames into a video file.
|
|
74
|
+
* Expects files named frame_00000.png, frame_00001.png, etc. in framesDir.
|
|
75
|
+
*/
|
|
76
|
+
export const framesToVideo = (framesDir, outputPath, fps, width, height) => {
|
|
77
|
+
const safe = normalizeEvenDimensions(width, height);
|
|
78
|
+
if (safe.width !== width || safe.height !== height) {
|
|
79
|
+
GraphAILogger.info(`framesToVideo: adjusted ${width}x${height} → ${safe.width}x${safe.height} (libx264 yuv420p requires even dimensions)`);
|
|
80
|
+
}
|
|
81
|
+
return new Promise((resolve, reject) => {
|
|
82
|
+
ffmpeg()
|
|
83
|
+
.input(`${framesDir}/frame_%05d.png`)
|
|
84
|
+
.inputOptions(["-framerate", String(fps)])
|
|
85
|
+
.outputOptions(["-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", String(fps), "-vf", `scale=${safe.width}:${safe.height}`])
|
|
86
|
+
.output(outputPath)
|
|
87
|
+
.on("end", () => resolve())
|
|
88
|
+
.on("error", (err) => reject(err))
|
|
89
|
+
.run();
|
|
90
|
+
});
|
|
91
|
+
};
|
|
65
92
|
export const ffmpegGetMediaDuration = (filePath) => {
|
|
66
93
|
return new Promise((resolve, reject) => {
|
|
67
94
|
// Only check file existence for local paths, not URLs
|
|
@@ -90,9 +117,13 @@ export const ffmpegGetMediaDuration = (filePath) => {
|
|
|
90
117
|
});
|
|
91
118
|
});
|
|
92
119
|
};
|
|
93
|
-
export const extractImageFromMovie = (movieFile, imagePath) => {
|
|
120
|
+
export const extractImageFromMovie = (movieFile, imagePath, useLastFrame = false) => {
|
|
94
121
|
return new Promise((resolve, reject) => {
|
|
95
|
-
ffmpeg(movieFile)
|
|
122
|
+
const command = ffmpeg(movieFile);
|
|
123
|
+
if (useLastFrame) {
|
|
124
|
+
command.inputOptions(["-sseof", "-0.1"]);
|
|
125
|
+
}
|
|
126
|
+
command
|
|
96
127
|
.outputOptions(["-frames:v 1"])
|
|
97
128
|
.output(imagePath)
|
|
98
129
|
.on("end", () => resolve({}))
|
package/lib/utils/file.d.ts
CHANGED
|
@@ -30,6 +30,7 @@ export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: n
|
|
|
30
30
|
imagePath: string;
|
|
31
31
|
htmlImageFile: string;
|
|
32
32
|
};
|
|
33
|
+
export declare const getBeatAnimatedVideoPath: (context: MulmoStudioContext, index: number) => string;
|
|
33
34
|
export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
|
|
34
35
|
movieFile: string;
|
|
35
36
|
soundEffectFile: string;
|
package/lib/utils/file.js
CHANGED
|
@@ -99,6 +99,12 @@ export const getBeatPngImagePath = (context, index) => {
|
|
|
99
99
|
const htmlImageFile = `${imageProjectDirPath}/${filename}_html.png`;
|
|
100
100
|
return { imagePath, htmlImageFile };
|
|
101
101
|
};
|
|
102
|
+
export const getBeatAnimatedVideoPath = (context, index) => {
|
|
103
|
+
const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
|
|
104
|
+
const beat = context.studio.script.beats[index];
|
|
105
|
+
const filename = beat?.id ? `${beat.id}` : `${index}${imageSuffix}`;
|
|
106
|
+
return `${imageProjectDirPath}/${filename}_animated.mp4`;
|
|
107
|
+
};
|
|
102
108
|
export const getBeatMoviePaths = (context, index) => {
|
|
103
109
|
const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
|
|
104
110
|
const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
|
|
@@ -1,3 +1,14 @@
|
|
|
1
1
|
export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean) => Promise<void>;
|
|
2
|
+
/**
|
|
3
|
+
* Render an animated HTML page frame-by-frame using Puppeteer.
|
|
4
|
+
*
|
|
5
|
+
* For each frame:
|
|
6
|
+
* 1. Update window.__MULMO.frame
|
|
7
|
+
* 2. Call render(frame, totalFrames, fps) — awaits if it returns a Promise
|
|
8
|
+
* 3. Take a screenshot
|
|
9
|
+
*
|
|
10
|
+
* The user-defined render() function may be sync or async.
|
|
11
|
+
*/
|
|
12
|
+
export declare const renderHTMLToFrames: (html: string, outputDir: string, width: number, height: number, totalFrames: number, fps: number) => Promise<string[]>;
|
|
2
13
|
export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number) => Promise<void>;
|
|
3
14
|
export declare const interpolate: (template: string, data: Record<string, string>) => string;
|