mulmocast 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -2
- package/assets/templates/business.json +2 -181
- package/assets/templates/children_book.json +1 -128
- package/assets/templates/coding.json +2 -136
- package/assets/templates/comic_strips.json +1 -30
- package/assets/templates/ghibli_strips.json +1 -30
- package/assets/templates/sensei_and_taro.json +1 -118
- package/lib/actions/audio.js +41 -31
- package/lib/actions/captions.js +39 -26
- package/lib/actions/images.js +31 -11
- package/lib/actions/movie.js +30 -21
- package/lib/actions/pdf.js +11 -1
- package/lib/actions/translate.js +33 -18
- package/lib/agents/combine_audio_files_agent.js +19 -8
- package/lib/cli/commands/tool/scripting/builder.d.ts +1 -1
- package/lib/cli/commands/tool/scripting/builder.js +4 -4
- package/lib/cli/commands/tool/scripting/handler.d.ts +2 -1
- package/lib/cli/commands/tool/scripting/handler.js +3 -3
- package/lib/cli/commands/tool/story_to_script/builder.d.ts +3 -1
- package/lib/cli/commands/tool/story_to_script/builder.js +12 -4
- package/lib/cli/commands/tool/story_to_script/handler.d.ts +3 -1
- package/lib/cli/commands/tool/story_to_script/handler.js +4 -3
- package/lib/methods/mulmo_script_template.d.ts +2 -2
- package/lib/methods/mulmo_script_template.js +2 -2
- package/lib/methods/mulmo_studio.d.ts +8 -0
- package/lib/methods/mulmo_studio.js +24 -0
- package/lib/tools/create_mulmo_script_from_url.d.ts +1 -1
- package/lib/tools/create_mulmo_script_from_url.js +7 -7
- package/lib/tools/create_mulmo_script_interactively.d.ts +1 -1
- package/lib/tools/create_mulmo_script_interactively.js +8 -8
- package/lib/tools/story_to_script.d.ts +5 -3
- package/lib/tools/story_to_script.js +90 -16
- package/lib/types/schema.d.ts +320 -1766
- package/lib/types/schema.js +41 -2
- package/lib/types/type.d.ts +4 -2
- package/lib/utils/const.d.ts +4 -0
- package/lib/utils/const.js +4 -0
- package/lib/utils/file.d.ts +1 -0
- package/lib/utils/file.js +16 -4
- package/lib/utils/filters.js +16 -11
- package/lib/utils/markdown.js +0 -2
- package/lib/utils/preprocess.d.ts +34 -15
- package/lib/utils/preprocess.js +3 -2
- package/lib/utils/prompt.d.ts +2 -1
- package/lib/utils/prompt.js +20 -3
- package/lib/utils/utils.d.ts +8 -5
- package/lib/utils/utils.js +27 -17
- package/package.json +3 -2
- package/scripts/templates/business.json +201 -0
- package/scripts/templates/children_book.json +97 -0
- package/scripts/templates/coding.json +130 -0
- package/scripts/templates/comic_strips.json +30 -0
- package/scripts/templates/ghibli_strips.json +30 -0
- package/scripts/templates/sensei_and_taro.json +127 -0
package/lib/types/schema.js
CHANGED
|
@@ -148,10 +148,16 @@ export const videoParamsSchema = z
|
|
|
148
148
|
})
|
|
149
149
|
.strict();
|
|
150
150
|
*/
|
|
151
|
+
export const beatAudioParamsSchema = z
|
|
152
|
+
.object({
|
|
153
|
+
padding: z.number().optional().describe("Padding between beats"), // seconds
|
|
154
|
+
})
|
|
155
|
+
.strict();
|
|
156
|
+
// Note: we can't extend beatAudioParamsSchema because it has padding as optional
|
|
151
157
|
export const audioParamsSchema = z
|
|
152
158
|
.object({
|
|
153
|
-
introPadding: z.number().describe("Padding at the beginning of the audio"), // seconds
|
|
154
159
|
padding: z.number().describe("Padding between beats"), // seconds
|
|
160
|
+
introPadding: z.number().describe("Padding at the beginning of the audio"), // seconds
|
|
155
161
|
closingPadding: z.number().describe("Padding before the last beat"), // seconds
|
|
156
162
|
outroPadding: z.number().describe("Padding at the end of the audio"), // seconds
|
|
157
163
|
})
|
|
@@ -164,6 +170,7 @@ export const mulmoBeatSchema = z
|
|
|
164
170
|
audio: mulmoAudioAssetSchema.optional(),
|
|
165
171
|
duration: z.number().optional().describe("Duration of the beat. Used only when the text is empty"),
|
|
166
172
|
imageParams: mulmoImageParamsSchema.optional(), // beat specific parameters
|
|
173
|
+
audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
|
|
167
174
|
speechOptions: speechOptionsSchema.optional(),
|
|
168
175
|
textSlideParams: textSlideParamsSchema.optional(),
|
|
169
176
|
imagePrompt: z.string().optional(), // specified or inserted by preprocessor
|
|
@@ -252,12 +259,44 @@ export const mulmoStudioMultiLingualDataSchema = z.object({
|
|
|
252
259
|
multiLingualTexts: multiLingualTextsSchema,
|
|
253
260
|
});
|
|
254
261
|
export const mulmoStudioMultiLingualSchema = z.array(mulmoStudioMultiLingualDataSchema).min(1);
|
|
262
|
+
export const mulmoSessionStateSchema = z.object({
|
|
263
|
+
inSession: z.object({
|
|
264
|
+
audio: z.boolean(),
|
|
265
|
+
image: z.boolean(),
|
|
266
|
+
video: z.boolean(),
|
|
267
|
+
multiLingual: z.boolean(),
|
|
268
|
+
caption: z.boolean(),
|
|
269
|
+
pdf: z.boolean(),
|
|
270
|
+
}),
|
|
271
|
+
inBeatSession: z.object({
|
|
272
|
+
audio: z.set(z.number()),
|
|
273
|
+
image: z.set(z.number()),
|
|
274
|
+
multiLingual: z.set(z.number()),
|
|
275
|
+
caption: z.set(z.number()),
|
|
276
|
+
}),
|
|
277
|
+
});
|
|
255
278
|
export const mulmoStudioSchema = z
|
|
256
279
|
.object({
|
|
257
280
|
script: mulmoScriptSchema,
|
|
258
281
|
filename: z.string(),
|
|
259
282
|
beats: z.array(mulmoStudioBeatSchema).min(1),
|
|
260
283
|
multiLingual: mulmoStudioMultiLingualSchema,
|
|
284
|
+
state: mulmoSessionStateSchema.default({
|
|
285
|
+
inSession: {
|
|
286
|
+
audio: false,
|
|
287
|
+
image: false,
|
|
288
|
+
video: false,
|
|
289
|
+
multiLingual: false,
|
|
290
|
+
caption: false,
|
|
291
|
+
pdf: false,
|
|
292
|
+
},
|
|
293
|
+
inBeatSession: {
|
|
294
|
+
audio: new Set(),
|
|
295
|
+
image: new Set(),
|
|
296
|
+
multiLingual: new Set(),
|
|
297
|
+
caption: new Set(),
|
|
298
|
+
},
|
|
299
|
+
}),
|
|
261
300
|
})
|
|
262
301
|
.strict();
|
|
263
302
|
export const mulmoScriptTemplateSchema = z
|
|
@@ -265,7 +304,7 @@ export const mulmoScriptTemplateSchema = z
|
|
|
265
304
|
title: z.string(),
|
|
266
305
|
description: z.string(),
|
|
267
306
|
systemPrompt: z.string(),
|
|
268
|
-
|
|
307
|
+
scriptName: z.string().optional(),
|
|
269
308
|
})
|
|
270
309
|
.strict();
|
|
271
310
|
export const mulmoStoryboardSceneSchema = z
|
package/lib/types/type.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, text2ImageProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema } from "./schema.js";
|
|
2
|
-
import { pdf_modes, pdf_sizes } from "../utils/const.js";
|
|
2
|
+
import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
|
|
3
|
+
import { LLM } from "../utils/utils.js";
|
|
3
4
|
import { z } from "zod";
|
|
4
5
|
export type LANG = z.infer<typeof langSchema>;
|
|
5
6
|
export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
|
|
@@ -50,7 +51,7 @@ export type ScriptingParams = {
|
|
|
50
51
|
templateName: string;
|
|
51
52
|
filename: string;
|
|
52
53
|
llm_model?: string;
|
|
53
|
-
|
|
54
|
+
llm?: LLM;
|
|
54
55
|
};
|
|
55
56
|
export type ImageProcessorParams = {
|
|
56
57
|
beat: MulmoBeat;
|
|
@@ -67,3 +68,4 @@ export type Text2ImageAgentInfo = {
|
|
|
67
68
|
imageParams: MulmoImageParams;
|
|
68
69
|
};
|
|
69
70
|
export type BeatMediaType = "movie" | "image";
|
|
71
|
+
export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
|
package/lib/utils/const.d.ts
CHANGED
|
@@ -5,3 +5,7 @@ export declare const cacheDirName = "cache";
|
|
|
5
5
|
export declare const pdf_modes: string[];
|
|
6
6
|
export declare const pdf_sizes: string[];
|
|
7
7
|
export declare const languages: string[];
|
|
8
|
+
export declare const storyToScriptGenerateMode: {
|
|
9
|
+
stepWise: string;
|
|
10
|
+
oneStep: string;
|
|
11
|
+
};
|
package/lib/utils/const.js
CHANGED
|
@@ -5,3 +5,7 @@ export const cacheDirName = "cache";
|
|
|
5
5
|
export const pdf_modes = ["slide", "talk", "handout"];
|
|
6
6
|
export const pdf_sizes = ["letter", "a4"];
|
|
7
7
|
export const languages = ["en", "ja"];
|
|
8
|
+
export const storyToScriptGenerateMode = {
|
|
9
|
+
stepWise: "step_wise",
|
|
10
|
+
oneStep: "one_step",
|
|
11
|
+
};
|
package/lib/utils/file.d.ts
CHANGED
|
@@ -41,6 +41,7 @@ export declare const defaultBGMPath: string;
|
|
|
41
41
|
export declare const getHTMLFile: (filename: string) => string;
|
|
42
42
|
export declare const getBaseDirPath: (basedir?: string) => string;
|
|
43
43
|
export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
|
|
44
|
+
export declare const readScriptTemplateFile: (scriptName: string) => any;
|
|
44
45
|
export declare const readTemplatePrompt: (templateName: string) => string;
|
|
45
46
|
export declare const getAvailableTemplates: () => (MulmoScriptTemplate & {
|
|
46
47
|
filename: string;
|
package/lib/utils/file.js
CHANGED
|
@@ -111,11 +111,23 @@ export const getFullPath = (baseDirPath, file) => {
|
|
|
111
111
|
}
|
|
112
112
|
return path.resolve(file);
|
|
113
113
|
};
|
|
114
|
+
export const readScriptTemplateFile = (scriptName) => {
|
|
115
|
+
const scriptPath = path.resolve(__dirname, "../../scripts/templates", scriptName);
|
|
116
|
+
const scriptData = fs.readFileSync(scriptPath, "utf-8");
|
|
117
|
+
// NOTE: We don't want to schema parse the script here to eliminate default values.
|
|
118
|
+
return JSON.parse(scriptData);
|
|
119
|
+
};
|
|
114
120
|
export const readTemplatePrompt = (templateName) => {
|
|
115
121
|
const templatePath = getTemplateFilePath(templateName);
|
|
116
|
-
const
|
|
117
|
-
const template = mulmoScriptTemplateSchema.parse(JSON.parse(
|
|
118
|
-
const
|
|
122
|
+
const templateData = fs.readFileSync(templatePath, "utf-8");
|
|
123
|
+
const template = mulmoScriptTemplateSchema.parse(JSON.parse(templateData));
|
|
124
|
+
const script = (() => {
|
|
125
|
+
if (template.scriptName) {
|
|
126
|
+
return readScriptTemplateFile(template.scriptName);
|
|
127
|
+
}
|
|
128
|
+
return undefined;
|
|
129
|
+
})();
|
|
130
|
+
const prompt = MulmoScriptTemplateMethods.getSystemPrompt(template, script);
|
|
119
131
|
return prompt;
|
|
120
132
|
};
|
|
121
133
|
export const getAvailableTemplates = () => {
|
|
@@ -133,7 +145,7 @@ export const getAvailableTemplates = () => {
|
|
|
133
145
|
});
|
|
134
146
|
};
|
|
135
147
|
export const writingMessage = (filePath) => {
|
|
136
|
-
GraphAILogger.
|
|
148
|
+
GraphAILogger.debug(`writing: ${filePath}`);
|
|
137
149
|
};
|
|
138
150
|
export const resolveMediaSource = (source, context) => {
|
|
139
151
|
if (source.kind === "path") {
|
package/lib/utils/filters.js
CHANGED
|
@@ -5,9 +5,10 @@ import fsPromise from "fs/promises";
|
|
|
5
5
|
import { GraphAILogger } from "graphai";
|
|
6
6
|
import { writingMessage } from "./file.js";
|
|
7
7
|
import { text2hash } from "./utils.js";
|
|
8
|
+
import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
|
|
8
9
|
export const fileCacheAgentFilter = async (context, next) => {
|
|
9
10
|
const { namedInputs } = context;
|
|
10
|
-
const { file,
|
|
11
|
+
const { file, force, studio, index, sessionType } = namedInputs;
|
|
11
12
|
const shouldUseCache = async () => {
|
|
12
13
|
if (force) {
|
|
13
14
|
return false;
|
|
@@ -21,19 +22,23 @@ export const fileCacheAgentFilter = async (context, next) => {
|
|
|
21
22
|
}
|
|
22
23
|
};
|
|
23
24
|
if (await shouldUseCache()) {
|
|
24
|
-
const elements = file.split("/");
|
|
25
|
-
GraphAILogger.info("cache hit: " + elements[elements.length - 1], text.slice(0, 10));
|
|
26
25
|
return true;
|
|
27
26
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
try {
|
|
28
|
+
MulmoStudioMethods.setBeatSessionState(studio, sessionType, index, true);
|
|
29
|
+
const output = (await next(context));
|
|
30
|
+
const buffer = output ? output["buffer"] : undefined;
|
|
31
|
+
if (buffer) {
|
|
32
|
+
writingMessage(file);
|
|
33
|
+
await fsPromise.writeFile(file, buffer);
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
GraphAILogger.log("no cache, no buffer: " + file);
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
MulmoStudioMethods.setBeatSessionState(studio, sessionType, index, false);
|
|
34
41
|
}
|
|
35
|
-
GraphAILogger.log("no cache, no buffer: " + file);
|
|
36
|
-
return false;
|
|
37
42
|
};
|
|
38
43
|
export const browserlessCacheGenerator = (cacheDir) => {
|
|
39
44
|
const browserlessCache = async (context, next) => {
|
package/lib/utils/markdown.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { GraphAILogger } from "graphai";
|
|
2
1
|
import { marked } from "marked";
|
|
3
2
|
import puppeteer from "puppeteer";
|
|
4
3
|
const isCI = process.env.CI === "true";
|
|
@@ -21,7 +20,6 @@ export const renderHTMLToImage = async (html, outputPath, width, height, isMerma
|
|
|
21
20
|
// Step 3: Capture screenshot of the page (which contains the Markdown-rendered HTML)
|
|
22
21
|
await page.screenshot({ path: outputPath, omitBackground: omitBackground });
|
|
23
22
|
await browser.close();
|
|
24
|
-
GraphAILogger.info(`HTML image rendered to ${outputPath}`);
|
|
25
23
|
};
|
|
26
24
|
export const renderMarkdownToImage = async (markdown, style, outputPath, width, height) => {
|
|
27
25
|
const header = `<head><style>${style}</style></head>`;
|
|
@@ -7,7 +7,22 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
|
|
|
7
7
|
imageFile?: string | undefined;
|
|
8
8
|
captionFile?: string | undefined;
|
|
9
9
|
}[];
|
|
10
|
+
multiLingual: {
|
|
11
|
+
multiLingualTexts: Record<string, {
|
|
12
|
+
text: string;
|
|
13
|
+
lang: string;
|
|
14
|
+
texts?: string[] | undefined;
|
|
15
|
+
ttsTexts?: string[] | undefined;
|
|
16
|
+
duration?: number | undefined;
|
|
17
|
+
}>;
|
|
18
|
+
}[];
|
|
10
19
|
script: {
|
|
20
|
+
audioParams: {
|
|
21
|
+
padding: number;
|
|
22
|
+
introPadding: number;
|
|
23
|
+
closingPadding: number;
|
|
24
|
+
outroPadding: number;
|
|
25
|
+
};
|
|
11
26
|
$mulmocast: {
|
|
12
27
|
version: "1.0";
|
|
13
28
|
credit?: "closing" | undefined;
|
|
@@ -27,12 +42,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
|
|
|
27
42
|
} | undefined;
|
|
28
43
|
}>;
|
|
29
44
|
};
|
|
30
|
-
audioParams: {
|
|
31
|
-
introPadding: number;
|
|
32
|
-
padding: number;
|
|
33
|
-
closingPadding: number;
|
|
34
|
-
outroPadding: number;
|
|
35
|
-
};
|
|
36
45
|
beats: {
|
|
37
46
|
text: string;
|
|
38
47
|
speaker: string;
|
|
@@ -164,6 +173,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
|
|
|
164
173
|
style?: string | undefined;
|
|
165
174
|
moderation?: string | undefined;
|
|
166
175
|
} | undefined;
|
|
176
|
+
audioParams?: {
|
|
177
|
+
padding?: number | undefined;
|
|
178
|
+
} | undefined;
|
|
167
179
|
textSlideParams?: {
|
|
168
180
|
cssStyles: string | string[];
|
|
169
181
|
} | undefined;
|
|
@@ -193,13 +205,20 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
|
|
|
193
205
|
__test_invalid__?: boolean | undefined;
|
|
194
206
|
};
|
|
195
207
|
filename: string;
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
208
|
+
state: {
|
|
209
|
+
inSession: {
|
|
210
|
+
pdf: boolean;
|
|
211
|
+
image: boolean;
|
|
212
|
+
audio: boolean;
|
|
213
|
+
video: boolean;
|
|
214
|
+
multiLingual: boolean;
|
|
215
|
+
caption: boolean;
|
|
216
|
+
};
|
|
217
|
+
inBeatSession: {
|
|
218
|
+
image: Set<number>;
|
|
219
|
+
audio: Set<number>;
|
|
220
|
+
multiLingual: Set<number>;
|
|
221
|
+
caption: Set<number>;
|
|
222
|
+
};
|
|
223
|
+
};
|
|
205
224
|
};
|
package/lib/utils/preprocess.js
CHANGED
|
@@ -4,12 +4,13 @@ const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
|
|
|
4
4
|
if (parsed.success) {
|
|
5
5
|
return parsed.data;
|
|
6
6
|
}
|
|
7
|
-
|
|
7
|
+
// We need to parse it to fill default values
|
|
8
|
+
return mulmoStudioSchema.parse({
|
|
8
9
|
script: mulmoScript,
|
|
9
10
|
filename: fileName,
|
|
10
11
|
beats: [...Array(mulmoScript.beats.length)].map(() => ({})),
|
|
11
12
|
multiLingual: [...Array(mulmoScript.beats.length)].map(() => ({ multiLingualTexts: {} })),
|
|
12
|
-
};
|
|
13
|
+
});
|
|
13
14
|
};
|
|
14
15
|
const mulmoCredit = (speaker) => {
|
|
15
16
|
return {
|
package/lib/utils/prompt.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { MulmoBeat, MulmoScript, MulmoScriptTemplate, MulmoStoryboard } from "..
|
|
|
2
2
|
export declare const imagePrompt: (beat: MulmoBeat, style?: string) => string;
|
|
3
3
|
export declare const graphDataScriptFromUrlPrompt: (sourceTextInput: string) => string;
|
|
4
4
|
export declare const graphDataScriptGeneratePrompt: (scene: string) => string;
|
|
5
|
-
export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoScriptTemplate) => string;
|
|
5
|
+
export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoScriptTemplate, script?: MulmoScript) => string;
|
|
6
6
|
export declare const interactiveClarificationPrompt = "If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.";
|
|
7
7
|
export declare const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
|
|
8
8
|
export declare const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";
|
|
@@ -13,3 +13,4 @@ export declare const sceneToBeatsPrompt: ({ sampleBeats, beatsPerScene, allScene
|
|
|
13
13
|
allScenes: string;
|
|
14
14
|
}) => string;
|
|
15
15
|
export declare const storyToScriptInfoPrompt: (scriptWithoutBeats: Omit<MulmoScript, "beats">, story: MulmoStoryboard) => string;
|
|
16
|
+
export declare const storyToScriptPrompt: (script: MulmoScript, beatsPerScene: number, story: MulmoStoryboard) => string;
|
package/lib/utils/prompt.js
CHANGED
|
@@ -10,10 +10,10 @@ export const graphDataScriptFromUrlPrompt = (sourceTextInput) => {
|
|
|
10
10
|
export const graphDataScriptGeneratePrompt = (scene) => {
|
|
11
11
|
return `Please generate a script from the following scenes: ${scene}`;
|
|
12
12
|
};
|
|
13
|
-
export const getMulmoScriptTemplateSystemPrompt = (template) => {
|
|
13
|
+
export const getMulmoScriptTemplateSystemPrompt = (template, script) => {
|
|
14
14
|
// script is provided, use it as a script template
|
|
15
|
-
if (
|
|
16
|
-
return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(
|
|
15
|
+
if (script) {
|
|
16
|
+
return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(script)}\n\`\`\``;
|
|
17
17
|
}
|
|
18
18
|
// script is not provided, use the default schema
|
|
19
19
|
const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
|
|
@@ -55,3 +55,20 @@ Only include keys that exist in the sample script.
|
|
|
55
55
|
Do not add any keys that are not present in the sample script.
|
|
56
56
|
Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`.trim();
|
|
57
57
|
};
|
|
58
|
+
export const storyToScriptPrompt = (script, beatsPerScene, story) => {
|
|
59
|
+
return `Generate script for the given storyboard, following the structure of the sample scripts below.
|
|
60
|
+
Storyboard:
|
|
61
|
+
- title: ${story.title}
|
|
62
|
+
- references: ${story.references?.map((reference) => JSON.stringify(reference)).join("\n")}
|
|
63
|
+
- scenes: ${JSON.stringify(story.scenes)}
|
|
64
|
+
|
|
65
|
+
Sample script:
|
|
66
|
+
\`\`\`JSON
|
|
67
|
+
${JSON.stringify(script)}
|
|
68
|
+
\`\`\`
|
|
69
|
+
|
|
70
|
+
Generate exactly ${beatsPerScene} scripts (beats) for each scene.
|
|
71
|
+
Only include keys that exist in the sample script.
|
|
72
|
+
Do not add any keys that are not present in the sample script.
|
|
73
|
+
Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`.trim();
|
|
74
|
+
};
|
package/lib/utils/utils.d.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
|
|
2
|
-
export declare const
|
|
3
|
-
type
|
|
4
|
-
export declare const
|
|
5
|
-
|
|
2
|
+
export declare const llm: readonly ["openAI", "anthropic", "gemini", "groq"];
|
|
3
|
+
export type LLM = (typeof llm)[number];
|
|
4
|
+
export declare const llmConfig: Record<LLM, {
|
|
5
|
+
agent: string;
|
|
6
|
+
defaultModel: string;
|
|
7
|
+
max_tokens: number;
|
|
8
|
+
}>;
|
|
9
|
+
export declare const llmPair: (_llm?: LLM, _model?: string) => {
|
|
6
10
|
agent: string;
|
|
7
11
|
model: string;
|
|
8
12
|
max_tokens: number;
|
|
@@ -11,4 +15,3 @@ export declare const chunkArray: <T>(array: T[], size?: number) => T[][];
|
|
|
11
15
|
export declare const isHttp: (fileOrUrl: string) => boolean;
|
|
12
16
|
export declare const text2hash: (input: string) => string;
|
|
13
17
|
export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoStudioMultiLingualData, lang?: string) => string;
|
|
14
|
-
export {};
|
package/lib/utils/utils.js
CHANGED
|
@@ -1,22 +1,32 @@
|
|
|
1
1
|
import * as crypto from "crypto";
|
|
2
|
-
export const
|
|
3
|
-
const
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
export const llm = ["openAI", "anthropic", "gemini", "groq"];
|
|
3
|
+
export const llmConfig = {
|
|
4
|
+
openAI: {
|
|
5
|
+
agent: "openAIAgent",
|
|
6
|
+
defaultModel: "gpt-4o",
|
|
7
|
+
max_tokens: 8192,
|
|
8
|
+
},
|
|
9
|
+
anthropic: {
|
|
10
|
+
agent: "anthropicAgent",
|
|
11
|
+
defaultModel: "claude-3-7-sonnet-20250219",
|
|
12
|
+
max_tokens: 8192,
|
|
13
|
+
},
|
|
14
|
+
gemini: {
|
|
15
|
+
agent: "geminiAgent",
|
|
16
|
+
defaultModel: "gemini-1.5-flash",
|
|
17
|
+
max_tokens: 8192,
|
|
18
|
+
},
|
|
19
|
+
groq: {
|
|
20
|
+
agent: "groqAgent",
|
|
21
|
+
defaultModel: "llama3-8b-8192",
|
|
22
|
+
max_tokens: 4096,
|
|
23
|
+
},
|
|
8
24
|
};
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
};
|
|
15
|
-
export const defaultOpenAIModel = defaultModels["openAIAgent"];
|
|
16
|
-
export const llmPair = (_agent, _model) => {
|
|
17
|
-
const agent = _agent && llmAgents.includes(_agent ?? "") ? _agent : "openAIAgent";
|
|
18
|
-
const model = _model ?? defaultModels[agent ?? ""];
|
|
19
|
-
const max_tokens = longMaxTokens[agent];
|
|
25
|
+
export const llmPair = (_llm, _model) => {
|
|
26
|
+
const llmKey = _llm ?? "openAI";
|
|
27
|
+
const agent = llmConfig[llmKey]?.agent ?? llmConfig.openAI.agent;
|
|
28
|
+
const model = _model ?? llmConfig[llmKey]?.defaultModel ?? llmConfig.openAI.defaultModel;
|
|
29
|
+
const max_tokens = llmConfig[llmKey]?.max_tokens ?? llmConfig.openAI.max_tokens;
|
|
20
30
|
return { agent, model, max_tokens };
|
|
21
31
|
};
|
|
22
32
|
export const chunkArray = (array, size = 3) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/cli/bin.js",
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
},
|
|
10
10
|
"files": [
|
|
11
11
|
"./lib",
|
|
12
|
+
"./scripts/templates",
|
|
12
13
|
"./assets/music/StarsBeyondEx.mp3",
|
|
13
14
|
"./assets/audio/silent300.mp3",
|
|
14
15
|
"./assets/audio/silent800.mp3",
|
|
@@ -66,7 +67,7 @@
|
|
|
66
67
|
"dotenv": "^16.4.7",
|
|
67
68
|
"fluent-ffmpeg": "^2.1.3",
|
|
68
69
|
"google-auth-library": "^9.15.1",
|
|
69
|
-
"graphai": "^
|
|
70
|
+
"graphai": "^2.0.3",
|
|
70
71
|
"inquirer": "^12.6.1",
|
|
71
72
|
"marked": "^15.0.11",
|
|
72
73
|
"ora": "^8.2.0",
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"credit": "closing"
|
|
5
|
+
},
|
|
6
|
+
"title": "Sample Title",
|
|
7
|
+
"references": [
|
|
8
|
+
{
|
|
9
|
+
"url": "https://www.somegreatwebsite.com/article/123",
|
|
10
|
+
"title": "Title of the article we are referencing",
|
|
11
|
+
"type": "article"
|
|
12
|
+
}
|
|
13
|
+
],
|
|
14
|
+
"lang": "en",
|
|
15
|
+
"beats": [
|
|
16
|
+
{
|
|
17
|
+
"text": "Today we're exploring a fascinating concept that has shaped some of the most innovative companies and leaders of our time: the Reality Distortion Field.",
|
|
18
|
+
"image": {
|
|
19
|
+
"type": "textSlide",
|
|
20
|
+
"slide": {
|
|
21
|
+
"title": "This is the title of the presentation"
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"text": "This is a sample slide, which just displays the title and the presenter's name of this presentation.",
|
|
27
|
+
"image": {
|
|
28
|
+
"type": "textSlide",
|
|
29
|
+
"slide": {
|
|
30
|
+
"title": "This is the title of the presentation",
|
|
31
|
+
"subtitle": "Tom Johnson"
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"text": "The evolution of humans is a complex journey that spans millions of years, shaped by biology, environment, and culture. Here's a high-level summary of the key stages in human evolution",
|
|
37
|
+
"image": {
|
|
38
|
+
"type": "textSlide",
|
|
39
|
+
"slide": {
|
|
40
|
+
"title": "Human Evolution",
|
|
41
|
+
"bullets": [
|
|
42
|
+
"Early Primates",
|
|
43
|
+
"Hominids and Hominins",
|
|
44
|
+
"Australopithecus",
|
|
45
|
+
"Genus Homo Emerges",
|
|
46
|
+
"Homo erectus and Migration",
|
|
47
|
+
"Neanderthals and Other Archaic Humans",
|
|
48
|
+
"Homo sapiens"
|
|
49
|
+
]
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"text": "This is a table of items in the store.",
|
|
55
|
+
"image": {
|
|
56
|
+
"type": "markdown",
|
|
57
|
+
"markdown": [
|
|
58
|
+
"# Markdown Table Example",
|
|
59
|
+
"| Item | In Stock | Price |",
|
|
60
|
+
"| :---------------- | :------: | ----: |",
|
|
61
|
+
"| Python Hat | True | 23.99 |",
|
|
62
|
+
"| SQL Hat | True | 23.99 |",
|
|
63
|
+
"| Codecademy Tee | False | 19.99 |",
|
|
64
|
+
"| Codecademy Hoodie | False | 42.99 |"
|
|
65
|
+
]
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"text": "This page shows the sales and profits of this company from January 2024 to June 2024.",
|
|
70
|
+
"image": {
|
|
71
|
+
"type": "chart",
|
|
72
|
+
"title": "Sales and Profits (from Jan to June)",
|
|
73
|
+
"chartData": {
|
|
74
|
+
"type": "bar",
|
|
75
|
+
"data": {
|
|
76
|
+
"labels": ["January", "February", "March", "April", "May", "June"],
|
|
77
|
+
"datasets": [
|
|
78
|
+
{
|
|
79
|
+
"label": "Revenue ($1000s)",
|
|
80
|
+
"data": [120, 135, 180, 155, 170, 190],
|
|
81
|
+
"backgroundColor": "rgba(54, 162, 235, 0.5)",
|
|
82
|
+
"borderColor": "rgba(54, 162, 235, 1)",
|
|
83
|
+
"borderWidth": 1
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"label": "Profit ($1000s)",
|
|
87
|
+
"data": [45, 52, 68, 53, 61, 73],
|
|
88
|
+
"backgroundColor": "rgba(75, 192, 192, 0.5)",
|
|
89
|
+
"borderColor": "rgba(75, 192, 192, 1)",
|
|
90
|
+
"borderWidth": 1
|
|
91
|
+
}
|
|
92
|
+
]
|
|
93
|
+
},
|
|
94
|
+
"options": {
|
|
95
|
+
"responsive": true,
|
|
96
|
+
"animation": false
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"text": "This is a sample pie chart",
|
|
103
|
+
"image": {
|
|
104
|
+
"type": "chart",
|
|
105
|
+
"title": "A sample pie chart",
|
|
106
|
+
"chartData": {
|
|
107
|
+
"type": "pie",
|
|
108
|
+
"data": {
|
|
109
|
+
"labels": ["OpenAIと投資家の取り分", "マイクロソフトの取り分"],
|
|
110
|
+
"datasets": [
|
|
111
|
+
{
|
|
112
|
+
"data": [90, 10],
|
|
113
|
+
"backgroundColor": ["rgba(75, 192, 192, 0.5)", "rgba(54, 162, 235, 0.5)"],
|
|
114
|
+
"borderColor": ["rgba(75, 192, 192, 1)", "rgba(54, 162, 235, 1)"],
|
|
115
|
+
"borderWidth": 1
|
|
116
|
+
}
|
|
117
|
+
]
|
|
118
|
+
},
|
|
119
|
+
"options": {
|
|
120
|
+
"responsive": true,
|
|
121
|
+
"animation": false,
|
|
122
|
+
"plugins": {
|
|
123
|
+
"legend": {
|
|
124
|
+
"position": "bottom"
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"text": "Next, let's look at a diagram of our business process flow. This illustrates the key steps from product development to sales.",
|
|
133
|
+
"image": {
|
|
134
|
+
"type": "mermaid",
|
|
135
|
+
"title": "Business Process Flow",
|
|
136
|
+
"code": {
|
|
137
|
+
"kind": "text",
|
|
138
|
+
"text": "graph LR\n A[Market Research] --> B[Product Planning]\n B --> C[Development]\n C --> D[Testing]\n D --> E[Manufacturing]\n E --> F[Marketing]\n F --> G[Sales]\n G --> H[Customer Support]\n H --> A"
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"text": "This is a tailwind html format.",
|
|
144
|
+
"image": {
|
|
145
|
+
"type": "html_tailwind",
|
|
146
|
+
"html": [
|
|
147
|
+
"<main class=\"flex-grow\">",
|
|
148
|
+
" <!-- Hero Section -->",
|
|
149
|
+
" <section class=\"bg-blue-600 text-white py-20\">",
|
|
150
|
+
" <div class=\"container mx-auto px-6 text-center\">",
|
|
151
|
+
" <h1 class=\"text-4xl md:text-5xl font-bold mb-4\">Welcome to Mulmocast</h1>",
|
|
152
|
+
" <p class=\"text-lg md:text-xl mb-8\">A modern web experience powered by Tailwind CSS</p>",
|
|
153
|
+
" <a href=\"#features\" class=\"bg-white text-blue-600 px-6 py-3 rounded-lg font-semibold shadow hover:bg-gray-100 transition\">",
|
|
154
|
+
" Learn More",
|
|
155
|
+
" </a>",
|
|
156
|
+
" </div>",
|
|
157
|
+
" </section>",
|
|
158
|
+
"",
|
|
159
|
+
" <!-- Features Section -->",
|
|
160
|
+
" <section id=\"features\" class=\"py-16 bg-gray-100\">",
|
|
161
|
+
" <div class=\"container mx-auto px-6\">",
|
|
162
|
+
" <div class=\"grid grid-cols-1 md:grid-cols-3 gap-8 text-center\">",
|
|
163
|
+
" <div>",
|
|
164
|
+
" <div class=\"text-blue-600 text-4xl mb-2\">⚡</div>",
|
|
165
|
+
" <h3 class=\"text-xl font-semibold mb-2\">Fast</h3>",
|
|
166
|
+
" <p class=\"text-gray-600\">Built with performance in mind using modern tools.</p>",
|
|
167
|
+
" </div>",
|
|
168
|
+
" <div>",
|
|
169
|
+
" <div class=\"text-blue-600 text-4xl mb-2\">🎨</div>",
|
|
170
|
+
" <h3 class=\"text-xl font-semibold mb-2\">Beautiful</h3>",
|
|
171
|
+
" <p class=\"text-gray-600\">Styled with Tailwind CSS for clean, responsive design.</p>",
|
|
172
|
+
" </div>",
|
|
173
|
+
" <div>",
|
|
174
|
+
" <div class=\"text-blue-600 text-4xl mb-2\">🚀</div>",
|
|
175
|
+
" <h3 class=\"text-xl font-semibold mb-2\">Launch Ready</h3>",
|
|
176
|
+
" <p class=\"text-gray-600\">Easy to deploy and extend for your next big idea.</p>",
|
|
177
|
+
" </div>",
|
|
178
|
+
" </div>",
|
|
179
|
+
" </div>",
|
|
180
|
+
" </section>",
|
|
181
|
+
"</main>",
|
|
182
|
+
"",
|
|
183
|
+
"<!-- Footer -->",
|
|
184
|
+
"<footer class=\"bg-white text-gray-500 text-center py-6 border-t\">",
|
|
185
|
+
" 2025 Mulmocast.",
|
|
186
|
+
"</footer>"
|
|
187
|
+
]
|
|
188
|
+
}
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
"text": "This is the image of the future of enterprise applications.",
|
|
192
|
+
"image": {
|
|
193
|
+
"type": "image",
|
|
194
|
+
"source": {
|
|
195
|
+
"kind": "url",
|
|
196
|
+
"url": "https://satoshi.blogs.com/mag2/May2025/enterprise_app.png"
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
]
|
|
201
|
+
}
|