mulmocast 2.14.0 → 2.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/schemas/mulmo_script.json +2 -2
- package/lib/actions/avatar.d.ts +2 -1
- package/lib/actions/avatar.js +59 -3
- package/lib/index.remotion.d.ts +3 -0
- package/lib/index.remotion.js +2 -0
- package/lib/index.remotion_guide.d.ts +1 -0
- package/lib/index.remotion_guide.js +3 -0
- package/lib/types/schema.js +3 -1
- package/lib/utils/image_plugins/remotion.js +3 -10
- package/lib/utils/remotion/frames.d.ts +3 -0
- package/lib/utils/remotion/frames.js +11 -0
- package/lib/utils/remotion/render.d.ts +1 -1
- package/lib/utils/remotion/render.js +1 -1
- package/lib/utils/remotion/render_frames.d.ts +23 -0
- package/lib/utils/remotion/render_frames.js +61 -0
- package/package.json +7 -3
- package/scripts/test/test_avatar.json +97 -23
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"properties": {
|
|
126
126
|
"source": {
|
|
127
127
|
"type": "string",
|
|
128
|
-
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project)
|
|
128
|
+
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"
|
|
129
129
|
},
|
|
130
130
|
"position": {
|
|
131
131
|
"type": "object",
|
|
@@ -226,7 +226,7 @@
|
|
|
226
226
|
"properties": {
|
|
227
227
|
"source": {
|
|
228
228
|
"type": "string",
|
|
229
|
-
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project)
|
|
229
|
+
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"
|
|
230
230
|
},
|
|
231
231
|
"position": {
|
|
232
232
|
"type": "object",
|
package/lib/actions/avatar.d.ts
CHANGED
|
@@ -19,7 +19,7 @@ export type AvatarPlacementPlan = {
|
|
|
19
19
|
};
|
|
20
20
|
export type AvatarTrackPlan = {
|
|
21
21
|
speakerId: string;
|
|
22
|
-
/** absolute path
|
|
22
|
+
/** the avatar package: an absolute path, or an http(s) URL */
|
|
23
23
|
source: string;
|
|
24
24
|
segments: AvatarSegmentPlan[];
|
|
25
25
|
/** where the avatar is shown, beat by beat; it is not shown outside these */
|
|
@@ -33,5 +33,6 @@ export type AvatarTrackPlan = {
|
|
|
33
33
|
* avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
|
|
34
34
|
*/
|
|
35
35
|
export declare const planAvatarTracks: (context: MulmoStudioContext) => AvatarTrackPlan[];
|
|
36
|
+
export declare const avatarUrlStamp: (source: string) => Promise<(string | null)[]>;
|
|
36
37
|
/** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
|
|
37
38
|
export declare const avatar: (context: MulmoStudioContext) => Promise<MulmoStudioContext>;
|
package/lib/actions/avatar.js
CHANGED
|
@@ -7,7 +7,7 @@ import path from "node:path";
|
|
|
7
7
|
import { GraphAILogger } from "graphai";
|
|
8
8
|
import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
|
|
9
9
|
import { getFullPath, getOutputStudioFilePath } from "../utils/file.js";
|
|
10
|
-
import { localizedText } from "../utils/utils.js";
|
|
10
|
+
import { isHttp, localizedText } from "../utils/utils.js";
|
|
11
11
|
const DEFAULT_POSITION = { x: "84%", y: "100%", scale: "62%" };
|
|
12
12
|
// Show the whole avatar image. A rig may crop the top of its image (a flat cut edge, hidden when the
|
|
13
13
|
// avatar fills the frame); over a slide the cut would run through the head.
|
|
@@ -43,7 +43,7 @@ export const planAvatarTracks = (context) => {
|
|
|
43
43
|
tracks.set(found.speakerId, {
|
|
44
44
|
speakerId: found.speakerId,
|
|
45
45
|
avatar: found.avatar,
|
|
46
|
-
source: getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
|
|
46
|
+
source: isHttp(found.avatar.source) ? found.avatar.source : getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
|
|
47
47
|
segments: [],
|
|
48
48
|
placements: [],
|
|
49
49
|
duration,
|
|
@@ -105,6 +105,62 @@ const folderStamp = (dir) => {
|
|
|
105
105
|
.sort()
|
|
106
106
|
.map((file) => [path.relative(dir, file), fileStamp(file)]);
|
|
107
107
|
};
|
|
108
|
+
// An avatar at a URL: its small JSON files (avatar.json, the rig, layers.json, sprites.json) change
|
|
109
|
+
// whenever the avatar is edited or rebuilt, so they stand in for the files' stamps. They are fetched
|
|
110
|
+
// on every run; avatarscript caches the images.
|
|
111
|
+
// Redirects are followed only within the same origin, so a public host cannot send these requests
|
|
112
|
+
// to another one (an internal service, say). GitHub's github.com/…/raw/… links redirect to
|
|
113
|
+
// raw.githubusercontent.com: use the latter.
|
|
114
|
+
const MAX_REDIRECTS = 5;
|
|
115
|
+
const fetchSameOrigin = async (url) => {
|
|
116
|
+
let location = url;
|
|
117
|
+
for (let hops = 0; hops <= MAX_REDIRECTS; hops++) {
|
|
118
|
+
const response = await fetch(location, { redirect: "manual", signal: AbortSignal.timeout(60_000) });
|
|
119
|
+
const next = response.status >= 300 && response.status < 400 ? response.headers.get("location") : null;
|
|
120
|
+
if (!next)
|
|
121
|
+
return response;
|
|
122
|
+
const target = new URL(next, location);
|
|
123
|
+
if (target.origin !== url.origin)
|
|
124
|
+
throw new Error(`avatar: ${url.href} redirects to another host (${target.origin}); use the final URL`);
|
|
125
|
+
location = target;
|
|
126
|
+
}
|
|
127
|
+
throw new Error(`avatar: ${url.href}: too many redirects`);
|
|
128
|
+
};
|
|
129
|
+
const fetchText = async (url) => {
|
|
130
|
+
const response = await fetchSameOrigin(url);
|
|
131
|
+
if (response.status === 404)
|
|
132
|
+
return null;
|
|
133
|
+
if (!response.ok)
|
|
134
|
+
throw new Error(`avatar: ${url.href}: HTTP ${response.status}`);
|
|
135
|
+
return response.text();
|
|
136
|
+
};
|
|
137
|
+
export const avatarUrlStamp = async (source) => {
|
|
138
|
+
const url = new URL(source);
|
|
139
|
+
const folder = new URL(url);
|
|
140
|
+
folder.pathname = url.pathname.endsWith("/avatar.json") ? url.pathname.slice(0, -"avatar.json".length) : url.pathname.replace(/\/?$/, "/");
|
|
141
|
+
// the source's query (a version, a signature) goes with every file that has no query of its own
|
|
142
|
+
// like avatarscript, fetch only files inside the package's folder (no "../", no other host)
|
|
143
|
+
const at = (file, base) => {
|
|
144
|
+
const location = new URL(file, base);
|
|
145
|
+
if (location.origin !== folder.origin || !location.pathname.startsWith(folder.pathname)) {
|
|
146
|
+
throw new Error(`avatar: ${file} is outside the avatar package ${folder.href}`);
|
|
147
|
+
}
|
|
148
|
+
if (!location.search)
|
|
149
|
+
location.search = url.search;
|
|
150
|
+
return location;
|
|
151
|
+
};
|
|
152
|
+
const manifestText = await fetchText(at("avatar.json", folder));
|
|
153
|
+
const manifest = manifestText ? JSON.parse(manifestText) : undefined;
|
|
154
|
+
const root = at((manifest?.root ?? ".").replace(/\/?$/, "/"), folder);
|
|
155
|
+
// as avatarscript: a manifest names its files (sprites are optional); without one, a
|
|
156
|
+
// mesh-avatar-studio project's default paths
|
|
157
|
+
const assets = manifest ? (manifest.assets ?? {}) : { rig: "rig.json", layers: "built/layers.json", sprites: "built/sprites/sprites.json" };
|
|
158
|
+
const files = [assets.rig, assets.layers, assets.sprites];
|
|
159
|
+
// every location is checked before anything is fetched
|
|
160
|
+
const locations = files.map((file) => (file === undefined ? undefined : at(file, root)));
|
|
161
|
+
return [manifestText, ...(await Promise.all(locations.map((location) => (location ? fetchText(location) : null))))];
|
|
162
|
+
};
|
|
163
|
+
const sourceStamp = (source) => (isHttp(source) ? avatarUrlStamp(source) : folderStamp(source));
|
|
108
164
|
const even = (n) => Math.max(2, Math.round(n / 2) * 2); // video codecs need even sizes
|
|
109
165
|
/** Renders one track (or reuses it), returning where it goes on the canvas. */
|
|
110
166
|
const renderAvatarTrack = async (plan, context, getAvatarScript) => {
|
|
@@ -112,7 +168,7 @@ const renderAvatarTrack = async (plan, context, getAvatarScript) => {
|
|
|
112
168
|
// rendered once, at the largest size it is shown; smaller placements scale it down
|
|
113
169
|
const height = even(Math.max(...plan.placements.map((p) => percent(p.position.scale, canvas.height))));
|
|
114
170
|
const segments = plan.segments.map((segment) => ({ ...segment, audioStamp: fileStamp(segment.audio) }));
|
|
115
|
-
const identity = { version: 4, segments, source: plan.source, sourceStamp:
|
|
171
|
+
const identity = { version: 4, segments, source: plan.source, sourceStamp: await sourceStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
|
|
116
172
|
const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 12);
|
|
117
173
|
const dir = MulmoStudioContextMethods.getImageProjectDirPath(context);
|
|
118
174
|
const name = `avatar_${plan.speakerId.replace(/[^\w-]/g, "_")}_${hash}`;
|
package/lib/index.remotion.d.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
1
|
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
2
2
|
export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
|
|
3
3
|
export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";
|
|
4
|
+
export { REMOTION_REVIEW_FRACTIONS } from "./utils/remotion/frames.js";
|
|
5
|
+
export { renderRemotionFrames } from "./utils/remotion/render_frames.js";
|
|
6
|
+
export type { RemotionFrame, RemotionFramesRequest } from "./utils/remotion/render_frames.js";
|
package/lib/index.remotion.js
CHANGED
|
@@ -2,3 +2,5 @@
|
|
|
2
2
|
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
3
3
|
export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
|
|
4
4
|
export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";
|
|
5
|
+
export { REMOTION_REVIEW_FRACTIONS } from "./utils/remotion/frames.js";
|
|
6
|
+
export { renderRemotionFrames } from "./utils/remotion/render_frames.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
// Entry point for code that only needs the remotion component guide — a tool description built into
|
|
2
|
+
// a browser bundle, for example. `mulmocast/remotion` also carries the renderer, which needs Node.
|
|
3
|
+
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
package/lib/types/schema.js
CHANGED
|
@@ -46,7 +46,9 @@ export const avatarPositionSchema = z
|
|
|
46
46
|
export const avatarEmotionSchema = z.enum(["neutral", "happy", "sad", "angry", "surprised", "relaxed"]);
|
|
47
47
|
export const speakerAvatarSchema = z
|
|
48
48
|
.object({
|
|
49
|
-
source: z
|
|
49
|
+
source: z
|
|
50
|
+
.string()
|
|
51
|
+
.describe("AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"),
|
|
50
52
|
position: avatarPositionSchema.optional(),
|
|
51
53
|
})
|
|
52
54
|
.strict();
|
|
@@ -8,6 +8,7 @@ import { REMOTION_COMPONENT_FILE, buildRepairPrompt, buildReviewPrompt, buildSce
|
|
|
8
8
|
import { reviewComponentWithClaude, writeComponentWithClaude } from "../remotion/claude_runner.js";
|
|
9
9
|
import { ensureRemotionPackages, renderRemotionScene } from "../remotion/render.js";
|
|
10
10
|
import { commitFiles } from "../remotion/commit_files.js";
|
|
11
|
+
import { REMOTION_REVIEW_FRACTIONS, frameAtFraction, toFrameCount } from "../remotion/frames.js";
|
|
11
12
|
import { parrotingImagePath } from "./utils.js";
|
|
12
13
|
export const imageType = ImageMediaType.Remotion;
|
|
13
14
|
const DEFAULT_FPS = 30;
|
|
@@ -15,7 +16,6 @@ const MAX_REPAIR_ATTEMPTS = 2;
|
|
|
15
16
|
// Without audio the length is unknown; any length gives the same final frame, since the system
|
|
16
17
|
// prompt makes every animation finish before the end.
|
|
17
18
|
const STILL_ONLY_DURATION_SEC = 10;
|
|
18
|
-
const REVIEW_FRACTIONS = [0.3, 0.6, 0.95];
|
|
19
19
|
export const remotionWorkDir = (imagePath) => imagePath.replace(/(_animated)?\.(mp4|png)$/, "_remotion");
|
|
20
20
|
export const remotionStillPath = (imagePath) => imagePath.replace(/_animated\.mp4$/, ".png");
|
|
21
21
|
const loadOrWriteComponent = async (job, force, deps) => {
|
|
@@ -45,9 +45,9 @@ const renderWithRepair = async (job, code, attemptsLeft, deps, reviewStills = []
|
|
|
45
45
|
return await renderWithRepair(job, repaired, attemptsLeft - 1, deps, reviewStills);
|
|
46
46
|
}
|
|
47
47
|
};
|
|
48
|
-
const reviewStillsFor = (job) =>
|
|
48
|
+
const reviewStillsFor = (job) => REMOTION_REVIEW_FRACTIONS.map((fraction, index) => ({
|
|
49
49
|
fraction,
|
|
50
|
-
frame:
|
|
50
|
+
frame: frameAtFraction(job.props.durationInFrames, fraction),
|
|
51
51
|
file: nodePath.join(job.workDir, `review_${index}.png`),
|
|
52
52
|
}));
|
|
53
53
|
const errorMessage = (error) => (error instanceof Error ? error.message : String(error));
|
|
@@ -104,13 +104,6 @@ const reviewAndImprove = async (job, renderedCode, reviewStills, deps) => {
|
|
|
104
104
|
GraphAILogger.info(`remotion: visual review improved ${job.codePath}`);
|
|
105
105
|
await renderReviewedVersion(job, improved, deps);
|
|
106
106
|
};
|
|
107
|
-
const toFrameCount = (durationSec, fps) => {
|
|
108
|
-
const frames = Math.floor(durationSec * fps);
|
|
109
|
-
if (frames <= 0) {
|
|
110
|
-
throw new Error(`remotion: frame count is ${frames} (duration=${durationSec}, fps=${fps}). Increase duration or fps.`);
|
|
111
|
-
}
|
|
112
|
-
return frames;
|
|
113
|
-
};
|
|
114
107
|
// Lets a scene number itself and pace its place in the video; unknown when the beat is not in the script.
|
|
115
108
|
const scenePosition = (beats, index) => (index < 0 ? undefined : { index, count: beats.length });
|
|
116
109
|
// The words actually spoken over the scene: the translation when the video is rendered in another language.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// The same moments the visual review looks at; a host reviewing its own component gets these by default.
|
|
2
|
+
export const REMOTION_REVIEW_FRACTIONS = [0.3, 0.6, 0.95];
|
|
3
|
+
export const toFrameCount = (durationSec, fps) => {
|
|
4
|
+
const frames = Math.floor(durationSec * fps);
|
|
5
|
+
if (!Number.isFinite(frames) || frames <= 0) {
|
|
6
|
+
throw new Error(`remotion: frame count is ${frames} (duration=${durationSec}, fps=${fps}). Increase duration or fps.`);
|
|
7
|
+
}
|
|
8
|
+
return frames;
|
|
9
|
+
};
|
|
10
|
+
// Clamped to the last frame, so a fraction of 1 is the final frame rather than one past it.
|
|
11
|
+
export const frameAtFraction = (durationInFrames, fraction) => Math.min(durationInFrames - 1, Math.floor(durationInFrames * fraction));
|
|
@@ -57,7 +57,7 @@ export const renderRemotionScene = async ({ workDir, props, videoPath, stillPath
|
|
|
57
57
|
if (videoPath) {
|
|
58
58
|
await renderer.renderMedia({ composition, serveUrl, codec: "h264", outputLocation: videoPath, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
|
|
59
59
|
}
|
|
60
|
-
const stills = [...extraStills, { frame: composition.durationInFrames - 1, path: stillPath }];
|
|
60
|
+
const stills = stillPath ? [...extraStills, { frame: composition.durationInFrames - 1, path: stillPath }] : extraStills;
|
|
61
61
|
await stills.reduce(async (previous, still) => {
|
|
62
62
|
await previous;
|
|
63
63
|
await renderer.renderStill({ composition, serveUrl, output: still.path, frame: still.frame, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { SceneRenderer } from "./render.js";
|
|
2
|
+
export type RemotionFramesRequest = {
|
|
3
|
+
code: string;
|
|
4
|
+
durationSec: number;
|
|
5
|
+
outDir: string;
|
|
6
|
+
fps?: number;
|
|
7
|
+
width?: number;
|
|
8
|
+
height?: number;
|
|
9
|
+
fractions?: readonly number[];
|
|
10
|
+
};
|
|
11
|
+
export type RemotionFrame = {
|
|
12
|
+
fraction: number;
|
|
13
|
+
frame: number;
|
|
14
|
+
path: string;
|
|
15
|
+
};
|
|
16
|
+
export type RemotionFramesDeps = {
|
|
17
|
+
ensurePackages: () => void;
|
|
18
|
+
renderScene: SceneRenderer;
|
|
19
|
+
makeWorkDir: () => string;
|
|
20
|
+
};
|
|
21
|
+
export declare const framePlan: (durationInFrames: number, fractions: readonly number[], outDir: string) => RemotionFrame[];
|
|
22
|
+
export declare const createRemotionFrameRenderer: (deps: RemotionFramesDeps) => (request: RemotionFramesRequest) => Promise<RemotionFrame[]>;
|
|
23
|
+
export declare const renderRemotionFrames: (request: RemotionFramesRequest) => Promise<RemotionFrame[]>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { REMOTION_COMPONENT_FILE } from "./claude_prompt.js";
|
|
5
|
+
import { REMOTION_REVIEW_FRACTIONS, frameAtFraction, toFrameCount } from "./frames.js";
|
|
6
|
+
import { ensureRemotionPackages, renderRemotionScene } from "./render.js";
|
|
7
|
+
const DEFAULT_FPS = 30;
|
|
8
|
+
const DEFAULT_WIDTH = 1920;
|
|
9
|
+
const DEFAULT_HEIGHT = 1080;
|
|
10
|
+
const isValidFraction = (fraction) => Number.isFinite(fraction) && fraction >= 0 && fraction <= 1;
|
|
11
|
+
const checkFractions = (fractions) => {
|
|
12
|
+
if (fractions.length === 0)
|
|
13
|
+
throw new Error("remotion: fractions is empty");
|
|
14
|
+
const invalid = fractions.filter((fraction) => !isValidFraction(fraction));
|
|
15
|
+
if (invalid.length > 0)
|
|
16
|
+
throw new Error(`remotion: fractions must be between 0 and 1 (got ${invalid.join(", ")})`);
|
|
17
|
+
};
|
|
18
|
+
// Named by percent so a reader can tell which moment each file is; two fractions on one percent keep their order.
|
|
19
|
+
export const framePlan = (durationInFrames, fractions, outDir) => fractions.map((fraction, index) => {
|
|
20
|
+
const percent = String(Math.round(fraction * 100)).padStart(3, "0");
|
|
21
|
+
return { fraction, frame: frameAtFraction(durationInFrames, fraction), path: path.join(outDir, `frame_${index}_${percent}.png`) };
|
|
22
|
+
});
|
|
23
|
+
const isPositiveInteger = (value) => Number.isInteger(value) && value > 0;
|
|
24
|
+
const checkCanvas = (width, height) => {
|
|
25
|
+
if (!isPositiveInteger(width) || !isPositiveInteger(height))
|
|
26
|
+
throw new Error(`remotion: width and height must be positive integers (got ${width}x${height})`);
|
|
27
|
+
};
|
|
28
|
+
const errorMessage = (error) => (error instanceof Error ? error.message : String(error));
|
|
29
|
+
export const createRemotionFrameRenderer = (deps) => async (request) => {
|
|
30
|
+
const fps = request.fps ?? DEFAULT_FPS;
|
|
31
|
+
const props = {
|
|
32
|
+
durationInFrames: toFrameCount(request.durationSec, fps),
|
|
33
|
+
fps,
|
|
34
|
+
width: request.width ?? DEFAULT_WIDTH,
|
|
35
|
+
height: request.height ?? DEFAULT_HEIGHT,
|
|
36
|
+
};
|
|
37
|
+
const fractions = request.fractions ?? REMOTION_REVIEW_FRACTIONS;
|
|
38
|
+
checkFractions(fractions);
|
|
39
|
+
checkCanvas(props.width, props.height);
|
|
40
|
+
deps.ensurePackages();
|
|
41
|
+
const frames = framePlan(props.durationInFrames, fractions, request.outDir);
|
|
42
|
+
fs.mkdirSync(request.outDir, { recursive: true });
|
|
43
|
+
const workDir = deps.makeWorkDir();
|
|
44
|
+
try {
|
|
45
|
+
fs.writeFileSync(path.join(workDir, REMOTION_COMPONENT_FILE), request.code);
|
|
46
|
+
await deps.renderScene({ workDir, props, extraStills: frames.map(({ frame, path: file }) => ({ frame, path: file })) });
|
|
47
|
+
return frames;
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
throw new Error(`remotion: failed to render the component's frames: ${errorMessage(error)}`, { cause: error });
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
fs.rmSync(workDir, { recursive: true, force: true });
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
// The bundle and entry files go to a temporary directory, so outDir ends up holding only the PNGs.
|
|
57
|
+
export const renderRemotionFrames = createRemotionFrameRenderer({
|
|
58
|
+
ensurePackages: ensureRemotionPackages,
|
|
59
|
+
renderScene: renderRemotionScene,
|
|
60
|
+
makeWorkDir: () => fs.mkdtempSync(path.join(os.tmpdir(), "mulmocast-remotion-frames-")),
|
|
61
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.15.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.node.js",
|
|
@@ -20,6 +20,10 @@
|
|
|
20
20
|
"types": "./lib/index.remotion.d.ts",
|
|
21
21
|
"default": "./lib/index.remotion.js"
|
|
22
22
|
},
|
|
23
|
+
"./remotion/guide": {
|
|
24
|
+
"types": "./lib/index.remotion_guide.d.ts",
|
|
25
|
+
"default": "./lib/index.remotion_guide.js"
|
|
26
|
+
},
|
|
23
27
|
"./assets/*": "./assets/*",
|
|
24
28
|
"./scripts/*": "./scripts/*",
|
|
25
29
|
"./tools/complete_script": {
|
|
@@ -145,7 +149,7 @@
|
|
|
145
149
|
"@types/jsdom": "^28.0.3",
|
|
146
150
|
"@types/node": "^26.6.4",
|
|
147
151
|
"@types/yargs": "^17.0.35",
|
|
148
|
-
"avatarscript": "^0.2
|
|
152
|
+
"avatarscript": "^0.3.2",
|
|
149
153
|
"cross-env": "^10.1.0",
|
|
150
154
|
"esbuild": "^0.28.2",
|
|
151
155
|
"eslint": "^10.12.0",
|
|
@@ -175,7 +179,7 @@
|
|
|
175
179
|
"@remotion/shapes": "^4.0.533",
|
|
176
180
|
"@remotion/three": "^4.0.533",
|
|
177
181
|
"@remotion/transitions": "^4.0.533",
|
|
178
|
-
"avatarscript": "^0.2
|
|
182
|
+
"avatarscript": "^0.3.2",
|
|
179
183
|
"onnxruntime-node": "^1.30.0",
|
|
180
184
|
"react": "^19.2.0",
|
|
181
185
|
"react-dom": "^19.2.0",
|
|
@@ -1,46 +1,120 @@
|
|
|
1
1
|
{
|
|
2
|
-
"$mulmocast": {
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
3
5
|
"lang": "ja",
|
|
4
6
|
"title": "Talking avatar over slides",
|
|
5
|
-
"description": "
|
|
6
|
-
"avatarParams": {
|
|
7
|
+
"description": "Loads the sample avatar ani from mulmocast-media by URL. Needs the npm packages avatarscript and onnxruntime-node.",
|
|
8
|
+
"avatarParams": {
|
|
9
|
+
"position": {
|
|
10
|
+
"x": "82%",
|
|
11
|
+
"y": "100%",
|
|
12
|
+
"scale": "62%"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
7
15
|
"speechParams": {
|
|
8
16
|
"speakers": {
|
|
9
|
-
"
|
|
10
|
-
"provider": "
|
|
11
|
-
"voiceId": "
|
|
12
|
-
"
|
|
13
|
-
|
|
17
|
+
"Ani": {
|
|
18
|
+
"provider": "gemini",
|
|
19
|
+
"voiceId": "Leda",
|
|
20
|
+
"speechOptions": {
|
|
21
|
+
"instruction": "Speak in a slightly high-pitched, curt tone with sudden flustered shifts—like a tsundere anime girl."
|
|
22
|
+
},
|
|
23
|
+
"displayName": {
|
|
24
|
+
"ja": "アニ"
|
|
25
|
+
},
|
|
26
|
+
"avatar": {
|
|
27
|
+
"source": "https://raw.githubusercontent.com/receptron/mulmocast-media/main/avatars/ani"
|
|
28
|
+
}
|
|
14
29
|
}
|
|
15
30
|
}
|
|
16
31
|
},
|
|
17
32
|
"beats": [
|
|
18
33
|
{
|
|
19
|
-
"speaker": "
|
|
20
|
-
"text": "
|
|
21
|
-
"image": {
|
|
22
|
-
|
|
34
|
+
"speaker": "Ani",
|
|
35
|
+
"text": "べ、別にあなたのために説明するわけじゃないんだからね!今日は、スライドの上でしゃべるアバターを紹介してあげる。",
|
|
36
|
+
"image": {
|
|
37
|
+
"type": "markdown",
|
|
38
|
+
"markdown": ["# Talking avatars", "", "- MulmoScript + AvatarScript", "- スライドの上でアバターが話す"]
|
|
39
|
+
},
|
|
40
|
+
"avatarParams": {
|
|
41
|
+
"emotion": "happy",
|
|
42
|
+
"motions": [
|
|
43
|
+
{
|
|
44
|
+
"motion": "nod",
|
|
45
|
+
"at": "紹介"
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"speaker": "Ani",
|
|
52
|
+
"text": "このアバターはね、メッシュ・アバター・スタジオで、たった一枚のイラストから作ったの。それを、マルモキャストの上で動かしてるのよ。",
|
|
53
|
+
"image": {
|
|
54
|
+
"type": "markdown",
|
|
55
|
+
"markdown": ["# How it's made", "", "- mesh-avatar-studio: 1枚のイラスト → メッシュアバター", "- MulmoCast + AvatarScript: 台本 → 音声 → 口パク動画"]
|
|
56
|
+
},
|
|
57
|
+
"avatarParams": {
|
|
58
|
+
"emotion": "relaxed",
|
|
59
|
+
"motions": [
|
|
60
|
+
{
|
|
61
|
+
"motion": "nod",
|
|
62
|
+
"at": "マルモキャスト"
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
23
66
|
},
|
|
24
67
|
{
|
|
25
|
-
"speaker": "
|
|
26
|
-
"text": "
|
|
27
|
-
"image": {
|
|
28
|
-
|
|
68
|
+
"speaker": "Ani",
|
|
69
|
+
"text": "位置と大きさは、台本全体で決めておいて、ビートごとに変えられるの。ほら、大きくなったでしょ。ちゃんと見なさいよね!",
|
|
70
|
+
"image": {
|
|
71
|
+
"type": "markdown",
|
|
72
|
+
"markdown": ["# Position and size", "", "- 台本全体: `avatarParams.position`", "- ビートごとに上書き"]
|
|
73
|
+
},
|
|
74
|
+
"avatarParams": {
|
|
75
|
+
"emotion": "relaxed",
|
|
76
|
+
"position": {
|
|
77
|
+
"x": "76%",
|
|
78
|
+
"scale": "82%"
|
|
79
|
+
},
|
|
80
|
+
"motions": [
|
|
81
|
+
{
|
|
82
|
+
"motion": "tilt",
|
|
83
|
+
"at": "大きく"
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
}
|
|
29
87
|
},
|
|
30
88
|
{
|
|
31
|
-
"speaker": "
|
|
32
|
-
"text": "
|
|
89
|
+
"speaker": "Ani",
|
|
90
|
+
"text": "ちょ、ちょっと!今度は左に小さくされちゃったんだけど!",
|
|
33
91
|
"image": {
|
|
34
92
|
"type": "markdown",
|
|
35
93
|
"markdown": ["# Moved", "", "```json", "\"avatarParams\": {", " \"position\": { \"x\": \"18%\", \"scale\": \"48%\" }", "}", "```"]
|
|
36
94
|
},
|
|
37
|
-
"avatarParams": {
|
|
95
|
+
"avatarParams": {
|
|
96
|
+
"emotion": "surprised",
|
|
97
|
+
"position": {
|
|
98
|
+
"x": "18%",
|
|
99
|
+
"scale": "48%"
|
|
100
|
+
},
|
|
101
|
+
"motions": [
|
|
102
|
+
{
|
|
103
|
+
"motion": "surprise"
|
|
104
|
+
}
|
|
105
|
+
]
|
|
106
|
+
}
|
|
38
107
|
},
|
|
39
108
|
{
|
|
40
|
-
"speaker": "
|
|
41
|
-
"text": "
|
|
42
|
-
"image": {
|
|
43
|
-
|
|
109
|
+
"speaker": "Ani",
|
|
110
|
+
"text": "最後は元の位置に戻るわ。べ、別に、こんなことが簡単にできるって、自慢してるわけじゃないんだからね!",
|
|
111
|
+
"image": {
|
|
112
|
+
"type": "markdown",
|
|
113
|
+
"markdown": ["# 簡単…でしょ?"]
|
|
114
|
+
},
|
|
115
|
+
"avatarParams": {
|
|
116
|
+
"emotion": "angry"
|
|
117
|
+
}
|
|
44
118
|
}
|
|
45
119
|
]
|
|
46
120
|
}
|