mulmocast 2.12.1 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/schemas/mulmo_script.json +297 -0
- package/lib/actions/avatar.d.ts +37 -0
- package/lib/actions/avatar.js +176 -0
- package/lib/actions/image_agents.js +9 -9
- package/lib/actions/index.d.ts +1 -0
- package/lib/actions/index.js +1 -0
- package/lib/actions/movie.d.ts +1 -0
- package/lib/actions/movie.js +40 -2
- package/lib/agents/combine_audio_files_agent.js +2 -2
- package/lib/cli/commands/movie/handler.js +2 -2
- package/lib/index.remotion.d.ts +3 -0
- package/lib/index.remotion.js +4 -0
- package/lib/mcp/server.js +3 -3
- package/lib/methods/mulmo_beat.d.ts +1 -0
- package/lib/methods/mulmo_beat.js +3 -0
- package/lib/methods/mulmo_beat_node.d.ts +1 -0
- package/lib/methods/mulmo_presentation_style.d.ts +5 -1
- package/lib/methods/mulmo_presentation_style.js +8 -0
- package/lib/types/schema.d.ts +466 -4
- package/lib/types/schema.js +77 -0
- package/lib/types/type.d.ts +7 -1
- package/lib/utils/context.d.ts +178 -0
- package/lib/utils/image_plugins/index.js +2 -0
- package/lib/utils/image_plugins/remotion.d.ts +17 -0
- package/lib/utils/image_plugins/remotion.js +202 -0
- package/lib/utils/remotion/claude_prompt.d.ts +33 -0
- package/lib/utils/remotion/claude_prompt.js +111 -0
- package/lib/utils/remotion/claude_runner.d.ts +4 -0
- package/lib/utils/remotion/claude_runner.js +47 -0
- package/lib/utils/remotion/commit_files.d.ts +11 -0
- package/lib/utils/remotion/commit_files.js +37 -0
- package/lib/utils/remotion/packages.d.ts +5 -0
- package/lib/utils/remotion/packages.js +20 -0
- package/lib/utils/remotion/render.d.ts +20 -0
- package/lib/utils/remotion/render.js +65 -0
- package/lib/utils/remotion/system_prompt.d.ts +4 -0
- package/lib/utils/remotion/system_prompt.js +66 -0
- package/package.json +112 -20
- package/scripts/test/test_avatar.json +46 -0
- package/scripts/test/test_remotion.json +38 -0
package/lib/types/schema.js
CHANGED
|
@@ -34,6 +34,37 @@ export const speechOptionsSchema = z
|
|
|
34
34
|
const speakerIdSchema = z.string();
|
|
35
35
|
export const defaultSpeaker = "Presenter";
|
|
36
36
|
export const text2SpeechProviderSchema = z.enum(Object.keys(provider2TTSAgent)).default(defaultProviders.tts);
|
|
37
|
+
// Talking avatars (AvatarScript): a speaker can speak with an avatar shown over the beat visuals.
|
|
38
|
+
const avatarPercentSchema = z.string().regex(/^-?\d+(\.\d+)?%$/, 'a percentage such as "84%"');
|
|
39
|
+
export const avatarPositionSchema = z
|
|
40
|
+
.object({
|
|
41
|
+
x: avatarPercentSchema.optional().describe("Horizontal centre of the avatar, % of the canvas width. Default 84%"),
|
|
42
|
+
y: avatarPercentSchema.optional().describe("Bottom edge of the avatar, % of the canvas height. Default 100%"),
|
|
43
|
+
scale: avatarPercentSchema.optional().describe("Height of the avatar, % of the canvas height. Default 62%"),
|
|
44
|
+
})
|
|
45
|
+
.strict();
|
|
46
|
+
export const avatarEmotionSchema = z.enum(["neutral", "happy", "sad", "angry", "surprised", "relaxed"]);
|
|
47
|
+
export const speakerAvatarSchema = z
|
|
48
|
+
.object({
|
|
49
|
+
source: z.string().describe("AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project), relative to the script"),
|
|
50
|
+
position: avatarPositionSchema.optional(),
|
|
51
|
+
})
|
|
52
|
+
.strict();
|
|
53
|
+
export const mulmoAvatarParamsSchema = z
|
|
54
|
+
.object({
|
|
55
|
+
position: avatarPositionSchema.optional(),
|
|
56
|
+
emotion: avatarEmotionSchema.optional().describe("The avatar's expression during this beat"),
|
|
57
|
+
motions: z
|
|
58
|
+
.array(z
|
|
59
|
+
.object({
|
|
60
|
+
motion: z.string().describe("Motion name, e.g. nod, tilt, think, giggle, surprise"),
|
|
61
|
+
at: z.string().optional().describe("Words of the beat's text where the motion starts. Default: the start of the beat"),
|
|
62
|
+
})
|
|
63
|
+
.strict())
|
|
64
|
+
.optional(),
|
|
65
|
+
hidden: z.boolean().optional().describe("Hide the speaker's avatar during this beat"),
|
|
66
|
+
})
|
|
67
|
+
.strict();
|
|
37
68
|
export const speakerDataSchema = z
|
|
38
69
|
.object({
|
|
39
70
|
displayName: z.record(langSchema, z.string()).optional(),
|
|
@@ -44,6 +75,7 @@ export const speakerDataSchema = z
|
|
|
44
75
|
model: z.string().optional().describe("TTS model to use for this speaker"),
|
|
45
76
|
baseURL: z.string().optional(), // Azure/custom endpoint URL
|
|
46
77
|
apiVersion: z.string().optional(), // Azure API version (e.g., "2025-04-01-preview")
|
|
78
|
+
avatar: speakerAvatarSchema.optional().describe("Avatar shown over the beat visuals, speaking this speaker's lines"),
|
|
47
79
|
})
|
|
48
80
|
.strict();
|
|
49
81
|
export const speakerSchema = speakerDataSchema.extend({
|
|
@@ -139,6 +171,7 @@ export const ImageMediaType = {
|
|
|
139
171
|
MoviePrompt: "moviePrompt",
|
|
140
172
|
Beat: "beat",
|
|
141
173
|
VoiceOver: "voice_over",
|
|
174
|
+
Remotion: "remotion",
|
|
142
175
|
};
|
|
143
176
|
// Beat image types whose still derives from the movie or another beat — they cannot supply
|
|
144
177
|
// their own still for the $beatImage frame sentinel.
|
|
@@ -326,6 +359,34 @@ export const mulmoVisionMediaSchema = z
|
|
|
326
359
|
data: z.record(z.string(), z.any()),
|
|
327
360
|
})
|
|
328
361
|
.strict();
|
|
362
|
+
export const remotionCodeSourceSchema = z.discriminatedUnion("kind", [
|
|
363
|
+
z.object({ kind: z.literal("text"), text: z.string().min(1) }).strict(),
|
|
364
|
+
z.object({ kind: z.literal("path"), path: z.string().min(1).describe("Relative to the script") }).strict(),
|
|
365
|
+
]);
|
|
366
|
+
const remotionMediaShape = {
|
|
367
|
+
type: z.literal(ImageMediaType.Remotion),
|
|
368
|
+
fps: z.number().min(1).max(60).optional().describe("Frame rate of the rendered video. Default: 30."),
|
|
369
|
+
};
|
|
370
|
+
// Two strict shapes rather than one object with a refine, so the bundled JSON schema states "prompt or code, not both" too.
|
|
371
|
+
export const mulmoRemotionMediaSchema = z.union([
|
|
372
|
+
z
|
|
373
|
+
.object({
|
|
374
|
+
...remotionMediaShape,
|
|
375
|
+
prompt: z.string().min(1).describe("What the scene shows. Claude Code (claude -p) writes a Remotion component from it."),
|
|
376
|
+
})
|
|
377
|
+
.strict(),
|
|
378
|
+
z
|
|
379
|
+
.object({
|
|
380
|
+
...remotionMediaShape,
|
|
381
|
+
code: remotionCodeSourceSchema.describe("A finished Remotion component (TSX that default-exports the scene). It is rendered as is, without claude -p."),
|
|
382
|
+
})
|
|
383
|
+
.strict(),
|
|
384
|
+
]);
|
|
385
|
+
export const mulmoRemotionParamsSchema = z
|
|
386
|
+
.object({
|
|
387
|
+
brief: z.string().optional().describe("Whole-video art direction shared by every remotion beat (palette, type, mood), so all scenes look like one film."),
|
|
388
|
+
})
|
|
389
|
+
.strict();
|
|
329
390
|
export const mulmoImageAssetSchema = z.union([
|
|
330
391
|
mulmoMarkdownMediaSchema,
|
|
331
392
|
mulmoWebMediaSchema,
|
|
@@ -341,6 +402,7 @@ export const mulmoImageAssetSchema = z.union([
|
|
|
341
402
|
mulmoVoiceOverMediaSchema,
|
|
342
403
|
mulmoVisionMediaSchema,
|
|
343
404
|
mulmoSlideMediaSchema,
|
|
405
|
+
mulmoRemotionMediaSchema,
|
|
344
406
|
]);
|
|
345
407
|
const mulmoAudioMediaSchema = z
|
|
346
408
|
.object({
|
|
@@ -567,6 +629,7 @@ export const mulmoBeatSchema = z
|
|
|
567
629
|
.optional(),
|
|
568
630
|
soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
|
|
569
631
|
lipSyncParams: mulmoLipSyncParamsSchema.optional(),
|
|
632
|
+
avatarParams: mulmoAvatarParamsSchema.optional(),
|
|
570
633
|
htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
|
|
571
634
|
speechOptions: speechOptionsSchema.optional(),
|
|
572
635
|
textSlideParams: textSlideParamsSchema.optional(),
|
|
@@ -626,6 +689,7 @@ export const mulmoPresentationStyleSchema = z.object({
|
|
|
626
689
|
provider: defaultProviders.soundEffect,
|
|
627
690
|
}),
|
|
628
691
|
lipSyncParams: mulmoLipSyncParamsSchema.optional(),
|
|
692
|
+
avatarParams: mulmoAvatarParamsSchema.optional(),
|
|
629
693
|
htmlImageParams: mulmoHtmlImageParamsSchema
|
|
630
694
|
.extend({
|
|
631
695
|
provider: text2HtmlImageProviderSchema,
|
|
@@ -635,6 +699,8 @@ export const mulmoPresentationStyleSchema = z.object({
|
|
|
635
699
|
textSlideParams: textSlideParamsSchema.optional(),
|
|
636
700
|
// for slide plugin
|
|
637
701
|
slideParams: mulmoSlideParamsSchema.optional(),
|
|
702
|
+
// for remotion plugin
|
|
703
|
+
remotionParams: mulmoRemotionParamsSchema.optional(),
|
|
638
704
|
captionParams: mulmoCaptionParamsSchema.optional(),
|
|
639
705
|
audioParams: audioParamsSchema.default({
|
|
640
706
|
introPadding: 1.0,
|
|
@@ -730,11 +796,22 @@ export const mulmoSessionStateSchema = z.object({
|
|
|
730
796
|
lipSync: z.record(z.string(), z.boolean()),
|
|
731
797
|
}),
|
|
732
798
|
});
|
|
799
|
+
export const mulmoAvatarTrackSchema = z
|
|
800
|
+
.object({
|
|
801
|
+
speaker: z.string(),
|
|
802
|
+
file: z.string(), // see-through, avatar-sized video (WebM VP9 with alpha)
|
|
803
|
+
width: z.number(), // the size it was rendered at, pixels
|
|
804
|
+
height: z.number(),
|
|
805
|
+
// where it is shown: absolute [start, end) seconds, and its place and size on the canvas in pixels
|
|
806
|
+
placements: z.array(z.object({ start: z.number(), end: z.number(), x: z.number(), y: z.number(), width: z.number(), height: z.number() }).strict()),
|
|
807
|
+
})
|
|
808
|
+
.strict();
|
|
733
809
|
export const mulmoStudioSchema = z
|
|
734
810
|
.object({
|
|
735
811
|
script: mulmoScriptSchema,
|
|
736
812
|
filename: z.string(),
|
|
737
813
|
beats: z.array(mulmoStudioBeatSchema).min(1),
|
|
814
|
+
avatarTracks: z.array(mulmoAvatarTrackSchema).optional(),
|
|
738
815
|
})
|
|
739
816
|
.strict();
|
|
740
817
|
export const mulmoPromptTemplateSchema = z
|
package/lib/types/type.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type CallbackFunction } from "graphai";
|
|
2
2
|
import type { UsageCollectorAPI } from "./usage.js";
|
|
3
|
-
import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
|
|
3
|
+
import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoRemotionMediaSchema, remotionCodeSourceSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema, mulmoAvatarParamsSchema, avatarPositionSchema, speakerAvatarSchema, mulmoAvatarTrackSchema } from "./schema.js";
|
|
4
4
|
import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
|
|
5
5
|
import type { LLM } from "./provider2agent.js";
|
|
6
6
|
import { z } from "zod";
|
|
@@ -44,6 +44,10 @@ export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
|
|
|
44
44
|
export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
|
|
45
45
|
export type MulmoSoundEffectParams = z.infer<typeof mulmoSoundEffectParamsSchema>;
|
|
46
46
|
export type MulmoLipSyncParams = z.infer<typeof mulmoLipSyncParamsSchema>;
|
|
47
|
+
export type MulmoAvatarParams = z.infer<typeof mulmoAvatarParamsSchema>;
|
|
48
|
+
export type MulmoAvatarPosition = z.infer<typeof avatarPositionSchema>;
|
|
49
|
+
export type MulmoSpeakerAvatar = z.infer<typeof speakerAvatarSchema>;
|
|
50
|
+
export type MulmoAvatarTrack = z.infer<typeof mulmoAvatarTrackSchema>;
|
|
47
51
|
export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
|
|
48
52
|
export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
|
|
49
53
|
export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
|
|
@@ -61,6 +65,8 @@ export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
|
|
|
61
65
|
export type MulmoMovieMedia = z.infer<typeof mulmoMovieMediaSchema>;
|
|
62
66
|
export type MulmoChartMedia = z.infer<typeof mulmoChartMediaSchema>;
|
|
63
67
|
export type MulmoMermaidMedia = z.infer<typeof mulmoMermaidMediaSchema>;
|
|
68
|
+
export type MulmoRemotionMedia = z.infer<typeof mulmoRemotionMediaSchema>;
|
|
69
|
+
export type MulmoRemotionCodeSource = z.infer<typeof remotionCodeSourceSchema>;
|
|
64
70
|
export type MulmoSessionState = z.infer<typeof mulmoSessionStateSchema>;
|
|
65
71
|
export type MulmoStudioContext = {
|
|
66
72
|
fileDirs: FileObject;
|
package/lib/utils/context.d.ts
CHANGED
|
@@ -53,6 +53,14 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
53
53
|
model?: string | undefined;
|
|
54
54
|
baseURL?: string | undefined;
|
|
55
55
|
apiVersion?: string | undefined;
|
|
56
|
+
avatar?: {
|
|
57
|
+
source: string;
|
|
58
|
+
position?: {
|
|
59
|
+
x?: string | undefined;
|
|
60
|
+
y?: string | undefined;
|
|
61
|
+
scale?: string | undefined;
|
|
62
|
+
} | undefined;
|
|
63
|
+
} | undefined;
|
|
56
64
|
lang?: Record<string, {
|
|
57
65
|
voiceId: string;
|
|
58
66
|
displayName?: Record<string, string> | undefined;
|
|
@@ -68,6 +76,14 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
68
76
|
model?: string | undefined;
|
|
69
77
|
baseURL?: string | undefined;
|
|
70
78
|
apiVersion?: string | undefined;
|
|
79
|
+
avatar?: {
|
|
80
|
+
source: string;
|
|
81
|
+
position?: {
|
|
82
|
+
x?: string | undefined;
|
|
83
|
+
y?: string | undefined;
|
|
84
|
+
scale?: string | undefined;
|
|
85
|
+
} | undefined;
|
|
86
|
+
} | undefined;
|
|
71
87
|
}> | undefined;
|
|
72
88
|
}>;
|
|
73
89
|
};
|
|
@@ -532,6 +548,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
532
548
|
type: "vision";
|
|
533
549
|
style: string;
|
|
534
550
|
data: Record<string, any>;
|
|
551
|
+
} | {
|
|
552
|
+
prompt: string;
|
|
553
|
+
type: "remotion";
|
|
554
|
+
fps?: number | undefined;
|
|
555
|
+
} | {
|
|
556
|
+
code: {
|
|
557
|
+
kind: "text";
|
|
558
|
+
text: string;
|
|
559
|
+
} | {
|
|
560
|
+
kind: "path";
|
|
561
|
+
path: string;
|
|
562
|
+
};
|
|
563
|
+
type: "remotion";
|
|
564
|
+
fps?: number | undefined;
|
|
535
565
|
} | {
|
|
536
566
|
type: "slide";
|
|
537
567
|
slide: {
|
|
@@ -2371,6 +2401,19 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
2371
2401
|
provider?: string | undefined;
|
|
2372
2402
|
model?: string | undefined;
|
|
2373
2403
|
} | undefined;
|
|
2404
|
+
avatarParams?: {
|
|
2405
|
+
position?: {
|
|
2406
|
+
x?: string | undefined;
|
|
2407
|
+
y?: string | undefined;
|
|
2408
|
+
scale?: string | undefined;
|
|
2409
|
+
} | undefined;
|
|
2410
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
|
|
2411
|
+
motions?: {
|
|
2412
|
+
motion: string;
|
|
2413
|
+
at?: string | undefined;
|
|
2414
|
+
}[] | undefined;
|
|
2415
|
+
hidden?: boolean | undefined;
|
|
2416
|
+
} | undefined;
|
|
2374
2417
|
htmlImageParams?: {
|
|
2375
2418
|
model?: string | undefined;
|
|
2376
2419
|
} | undefined;
|
|
@@ -2474,6 +2517,19 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
2474
2517
|
provider?: string | undefined;
|
|
2475
2518
|
model?: string | undefined;
|
|
2476
2519
|
} | undefined;
|
|
2520
|
+
avatarParams?: {
|
|
2521
|
+
position?: {
|
|
2522
|
+
x?: string | undefined;
|
|
2523
|
+
y?: string | undefined;
|
|
2524
|
+
scale?: string | undefined;
|
|
2525
|
+
} | undefined;
|
|
2526
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
|
|
2527
|
+
motions?: {
|
|
2528
|
+
motion: string;
|
|
2529
|
+
at?: string | undefined;
|
|
2530
|
+
}[] | undefined;
|
|
2531
|
+
hidden?: boolean | undefined;
|
|
2532
|
+
} | undefined;
|
|
2477
2533
|
htmlImageParams?: {
|
|
2478
2534
|
provider: string;
|
|
2479
2535
|
model?: string | undefined;
|
|
@@ -2540,6 +2596,9 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
2540
2596
|
} | undefined;
|
|
2541
2597
|
} | undefined;
|
|
2542
2598
|
} | undefined;
|
|
2599
|
+
remotionParams?: {
|
|
2600
|
+
brief?: string | undefined;
|
|
2601
|
+
} | undefined;
|
|
2543
2602
|
captionParams?: {
|
|
2544
2603
|
lang?: string | undefined;
|
|
2545
2604
|
styles?: string[] | undefined;
|
|
@@ -2588,6 +2647,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
2588
2647
|
markdown?: string | undefined;
|
|
2589
2648
|
html?: string | undefined;
|
|
2590
2649
|
}[];
|
|
2650
|
+
avatarTracks?: {
|
|
2651
|
+
speaker: string;
|
|
2652
|
+
file: string;
|
|
2653
|
+
width: number;
|
|
2654
|
+
height: number;
|
|
2655
|
+
placements: {
|
|
2656
|
+
start: number;
|
|
2657
|
+
end: number;
|
|
2658
|
+
x: number;
|
|
2659
|
+
y: number;
|
|
2660
|
+
width: number;
|
|
2661
|
+
height: number;
|
|
2662
|
+
}[];
|
|
2663
|
+
}[] | undefined;
|
|
2591
2664
|
};
|
|
2592
2665
|
export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
|
|
2593
2666
|
export declare const getMultiLingual: (multilingualFilePath: string, beats: MulmoStudioBeat[]) => MulmoStudioMultiLingual;
|
|
@@ -2619,6 +2692,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
2619
2692
|
model?: string | undefined;
|
|
2620
2693
|
baseURL?: string | undefined;
|
|
2621
2694
|
apiVersion?: string | undefined;
|
|
2695
|
+
avatar?: {
|
|
2696
|
+
source: string;
|
|
2697
|
+
position?: {
|
|
2698
|
+
x?: string | undefined;
|
|
2699
|
+
y?: string | undefined;
|
|
2700
|
+
scale?: string | undefined;
|
|
2701
|
+
} | undefined;
|
|
2702
|
+
} | undefined;
|
|
2622
2703
|
lang?: Record<string, {
|
|
2623
2704
|
voiceId: string;
|
|
2624
2705
|
displayName?: Record<string, string> | undefined;
|
|
@@ -2634,6 +2715,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
2634
2715
|
model?: string | undefined;
|
|
2635
2716
|
baseURL?: string | undefined;
|
|
2636
2717
|
apiVersion?: string | undefined;
|
|
2718
|
+
avatar?: {
|
|
2719
|
+
source: string;
|
|
2720
|
+
position?: {
|
|
2721
|
+
x?: string | undefined;
|
|
2722
|
+
y?: string | undefined;
|
|
2723
|
+
scale?: string | undefined;
|
|
2724
|
+
} | undefined;
|
|
2725
|
+
} | undefined;
|
|
2637
2726
|
}> | undefined;
|
|
2638
2727
|
}>;
|
|
2639
2728
|
};
|
|
@@ -3098,6 +3187,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
3098
3187
|
type: "vision";
|
|
3099
3188
|
style: string;
|
|
3100
3189
|
data: Record<string, any>;
|
|
3190
|
+
} | {
|
|
3191
|
+
prompt: string;
|
|
3192
|
+
type: "remotion";
|
|
3193
|
+
fps?: number | undefined;
|
|
3194
|
+
} | {
|
|
3195
|
+
code: {
|
|
3196
|
+
kind: "text";
|
|
3197
|
+
text: string;
|
|
3198
|
+
} | {
|
|
3199
|
+
kind: "path";
|
|
3200
|
+
path: string;
|
|
3201
|
+
};
|
|
3202
|
+
type: "remotion";
|
|
3203
|
+
fps?: number | undefined;
|
|
3101
3204
|
} | {
|
|
3102
3205
|
type: "slide";
|
|
3103
3206
|
slide: {
|
|
@@ -4937,6 +5040,19 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
4937
5040
|
provider?: string | undefined;
|
|
4938
5041
|
model?: string | undefined;
|
|
4939
5042
|
} | undefined;
|
|
5043
|
+
avatarParams?: {
|
|
5044
|
+
position?: {
|
|
5045
|
+
x?: string | undefined;
|
|
5046
|
+
y?: string | undefined;
|
|
5047
|
+
scale?: string | undefined;
|
|
5048
|
+
} | undefined;
|
|
5049
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
|
|
5050
|
+
motions?: {
|
|
5051
|
+
motion: string;
|
|
5052
|
+
at?: string | undefined;
|
|
5053
|
+
}[] | undefined;
|
|
5054
|
+
hidden?: boolean | undefined;
|
|
5055
|
+
} | undefined;
|
|
4940
5056
|
htmlImageParams?: {
|
|
4941
5057
|
model?: string | undefined;
|
|
4942
5058
|
} | undefined;
|
|
@@ -5040,6 +5156,19 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5040
5156
|
provider?: string | undefined;
|
|
5041
5157
|
model?: string | undefined;
|
|
5042
5158
|
} | undefined;
|
|
5159
|
+
avatarParams?: {
|
|
5160
|
+
position?: {
|
|
5161
|
+
x?: string | undefined;
|
|
5162
|
+
y?: string | undefined;
|
|
5163
|
+
scale?: string | undefined;
|
|
5164
|
+
} | undefined;
|
|
5165
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
|
|
5166
|
+
motions?: {
|
|
5167
|
+
motion: string;
|
|
5168
|
+
at?: string | undefined;
|
|
5169
|
+
}[] | undefined;
|
|
5170
|
+
hidden?: boolean | undefined;
|
|
5171
|
+
} | undefined;
|
|
5043
5172
|
htmlImageParams?: {
|
|
5044
5173
|
provider: string;
|
|
5045
5174
|
model?: string | undefined;
|
|
@@ -5106,6 +5235,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5106
5235
|
} | undefined;
|
|
5107
5236
|
} | undefined;
|
|
5108
5237
|
} | undefined;
|
|
5238
|
+
remotionParams?: {
|
|
5239
|
+
brief?: string | undefined;
|
|
5240
|
+
} | undefined;
|
|
5109
5241
|
captionParams?: {
|
|
5110
5242
|
lang?: string | undefined;
|
|
5111
5243
|
styles?: string[] | undefined;
|
|
@@ -5154,6 +5286,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5154
5286
|
markdown?: string | undefined;
|
|
5155
5287
|
html?: string | undefined;
|
|
5156
5288
|
}[];
|
|
5289
|
+
avatarTracks?: {
|
|
5290
|
+
speaker: string;
|
|
5291
|
+
file: string;
|
|
5292
|
+
width: number;
|
|
5293
|
+
height: number;
|
|
5294
|
+
placements: {
|
|
5295
|
+
start: number;
|
|
5296
|
+
end: number;
|
|
5297
|
+
x: number;
|
|
5298
|
+
y: number;
|
|
5299
|
+
width: number;
|
|
5300
|
+
height: number;
|
|
5301
|
+
}[];
|
|
5302
|
+
}[] | undefined;
|
|
5157
5303
|
};
|
|
5158
5304
|
multiLingual: {
|
|
5159
5305
|
multiLingualTexts: Record<string, {
|
|
@@ -5192,6 +5338,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5192
5338
|
model?: string | undefined;
|
|
5193
5339
|
baseURL?: string | undefined;
|
|
5194
5340
|
apiVersion?: string | undefined;
|
|
5341
|
+
avatar?: {
|
|
5342
|
+
source: string;
|
|
5343
|
+
position?: {
|
|
5344
|
+
x?: string | undefined;
|
|
5345
|
+
y?: string | undefined;
|
|
5346
|
+
scale?: string | undefined;
|
|
5347
|
+
} | undefined;
|
|
5348
|
+
} | undefined;
|
|
5195
5349
|
lang?: Record<string, {
|
|
5196
5350
|
voiceId: string;
|
|
5197
5351
|
displayName?: Record<string, string> | undefined;
|
|
@@ -5207,6 +5361,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5207
5361
|
model?: string | undefined;
|
|
5208
5362
|
baseURL?: string | undefined;
|
|
5209
5363
|
apiVersion?: string | undefined;
|
|
5364
|
+
avatar?: {
|
|
5365
|
+
source: string;
|
|
5366
|
+
position?: {
|
|
5367
|
+
x?: string | undefined;
|
|
5368
|
+
y?: string | undefined;
|
|
5369
|
+
scale?: string | undefined;
|
|
5370
|
+
} | undefined;
|
|
5371
|
+
} | undefined;
|
|
5210
5372
|
}> | undefined;
|
|
5211
5373
|
}>;
|
|
5212
5374
|
};
|
|
@@ -5498,6 +5660,19 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5498
5660
|
provider?: string | undefined;
|
|
5499
5661
|
model?: string | undefined;
|
|
5500
5662
|
} | undefined;
|
|
5663
|
+
avatarParams?: {
|
|
5664
|
+
position?: {
|
|
5665
|
+
x?: string | undefined;
|
|
5666
|
+
y?: string | undefined;
|
|
5667
|
+
scale?: string | undefined;
|
|
5668
|
+
} | undefined;
|
|
5669
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
|
|
5670
|
+
motions?: {
|
|
5671
|
+
motion: string;
|
|
5672
|
+
at?: string | undefined;
|
|
5673
|
+
}[] | undefined;
|
|
5674
|
+
hidden?: boolean | undefined;
|
|
5675
|
+
} | undefined;
|
|
5501
5676
|
htmlImageParams?: {
|
|
5502
5677
|
provider: string;
|
|
5503
5678
|
model?: string | undefined;
|
|
@@ -5564,6 +5739,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
5564
5739
|
} | undefined;
|
|
5565
5740
|
} | undefined;
|
|
5566
5741
|
} | undefined;
|
|
5742
|
+
remotionParams?: {
|
|
5743
|
+
brief?: string | undefined;
|
|
5744
|
+
} | undefined;
|
|
5567
5745
|
captionParams?: {
|
|
5568
5746
|
lang?: string | undefined;
|
|
5569
5747
|
styles?: string[] | undefined;
|
|
@@ -9,6 +9,7 @@ import * as pluginBeat from "./beat.js";
|
|
|
9
9
|
import * as pluginVoiceOver from "./voice_over.js";
|
|
10
10
|
import * as pluginVision from "./vision.js";
|
|
11
11
|
import * as pluginSlide from "./slide.js";
|
|
12
|
+
import * as pluginRemotion from "./remotion.js";
|
|
12
13
|
const imagePlugins = [
|
|
13
14
|
pluginTextSlide,
|
|
14
15
|
pluginMarkdown,
|
|
@@ -21,6 +22,7 @@ const imagePlugins = [
|
|
|
21
22
|
pluginVoiceOver,
|
|
22
23
|
pluginVision,
|
|
23
24
|
pluginSlide,
|
|
25
|
+
pluginRemotion,
|
|
24
26
|
];
|
|
25
27
|
export const findImagePlugin = (imageType) => {
|
|
26
28
|
return imagePlugins.find((plugin) => plugin.imageType === imageType);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { ImageProcessorParams, MulmoRemotionCodeSource, MulmoStudioContext } from "../../types/index.js";
|
|
2
|
+
import { ComponentReviewer, ComponentWriter } from "../remotion/claude_runner.js";
|
|
3
|
+
import { SceneRenderer } from "../remotion/render.js";
|
|
4
|
+
export declare const imageType: "remotion";
|
|
5
|
+
export type RemotionDeps = {
|
|
6
|
+
ensurePackages: () => void;
|
|
7
|
+
writeComponent: ComponentWriter;
|
|
8
|
+
reviewComponent: ComponentReviewer;
|
|
9
|
+
renderScene: SceneRenderer;
|
|
10
|
+
};
|
|
11
|
+
export declare const remotionWorkDir: (imagePath: string) => string;
|
|
12
|
+
export declare const remotionStillPath: (imagePath: string) => string;
|
|
13
|
+
export declare const reviewCandidatePath: (filePath: string) => string;
|
|
14
|
+
export declare const remotionCodeLocation: (code: MulmoRemotionCodeSource, context: MulmoStudioContext, beatIndex: number) => string;
|
|
15
|
+
export declare const createRemotionProcess: (deps: RemotionDeps) => (params: ImageProcessorParams) => Promise<string | undefined>;
|
|
16
|
+
export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;
|
|
17
|
+
export declare const path: (params: import("../../types/type.js").BeatPathParams) => string;
|