mulmocast 0.0.5 → 0.0.7

package/lib/cli/commands/tool/story_to_script/builder.js

@@ -0,0 +1,61 @@
+import { getAvailableTemplates } from "../../../../utils/file.js";
+import { llm } from "../../../../utils/utils.js";
+import { storyToScriptGenerateMode } from "../../../../utils/const.js";
+const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+export const builder = (yargs) => {
+    return yargs
+        .option("o", {
+        alias: "outdir",
+        description: "output dir",
+        demandOption: false,
+        type: "string",
+    })
+        .option("b", {
+        alias: "basedir",
+        description: "base dir",
+        demandOption: false,
+        type: "string",
+    })
+        .option("t", {
+        alias: "template",
+        description: "Template name to use",
+        demandOption: false,
+        choices: availableTemplateNames,
+        type: "string",
+    })
+        .option("s", {
+        alias: "script",
+        description: "script filename",
+        demandOption: false,
+        default: "script",
+        type: "string",
+    })
+        .option("beats_per_scene", {
+        description: "beats per scene",
+        demandOption: false,
+        default: 3,
+        type: "number",
+    })
+        .option("llm", {
+        description: "llm",
+        demandOption: false,
+        choices: llm,
+        type: "string",
+    })
+        .option("llm_model", {
+        description: "llm model",
+        demandOption: false,
+        type: "string",
+    })
+        .option("mode", {
+        description: "story to script generation mode",
+        demandOption: false,
+        choices: Object.values(storyToScriptGenerateMode),
+        default: storyToScriptGenerateMode.stepWise,
+        type: "string",
+    })
+        .positional("file", {
+        description: "story file path",
+        type: "string",
+    });
+};

package/lib/cli/commands/tool/story_to_script/handler.d.ts

@@ -0,0 +1,13 @@
+import { ToolCliArgs } from "../../../../types/cli_types.js";
+import { LLM } from "../../../../utils/utils.js";
+export declare const handler: (argv: ToolCliArgs<{
+    o?: string;
+    b?: string;
+    t?: string;
+    s?: string;
+    beats_per_scene?: number;
+    file?: string;
+    llm?: LLM;
+    llm_model?: string;
+    mode?: string;
+}>) => Promise<void>;

package/lib/cli/commands/tool/story_to_script/handler.js

@@ -0,0 +1,36 @@
+import { selectTemplate } from "../../../../utils/inquirer.js";
+import { setGraphAILogger } from "../../../../cli/helpers.js";
+import { storyToScript } from "../../../../tools/story_to_script.js";
+import { mulmoStoryboardSchema } from "../../../../types/schema.js";
+import { getBaseDirPath, getFullPath, readAndParseJson } from "../../../../utils/file.js";
+import { outDirName } from "../../../../utils/const.js";
+export const handler = async (argv) => {
+    const { v: verbose, s: filename, file, o: outdir, b: basedir, beats_per_scene, llm, llm_model, mode } = argv;
+    let { t: template } = argv;
+    const baseDirPath = getBaseDirPath(basedir);
+    const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
+    if (!template) {
+        template = await selectTemplate();
+    }
+    setGraphAILogger(verbose, {
+        outdir: outDirPath,
+        basedir: baseDirPath,
+        template,
+        fileName: filename,
+        beatsPerScene: beats_per_scene,
+        storyFilePath: file,
+        llmAgent: llm,
+        llmModel: llm_model,
+    });
+    const parsedStory = readAndParseJson(file, mulmoStoryboardSchema);
+    await storyToScript({
+        story: parsedStory,
+        beatsPerScene: beats_per_scene,
+        templateName: template,
+        outdir: outDirPath,
+        fileName: filename,
+        llm,
+        llmModel: llm_model,
+        generateMode: mode,
+    });
+};

package/lib/cli/commands/tool/story_to_script/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "story_to_script <file>";
+export declare const desc = "Generate Mulmo script from story";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/tool/story_to_script/index.js

@@ -0,0 +1,4 @@
+export const command = "story_to_script <file>";
+export const desc = "Generate Mulmo script from story";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/translate/builder.d.ts

@@ -0,0 +1,14 @@
+import type { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<import("yargs").Omit<{
+    o: string | undefined;
+} & {
+    b: string | undefined;
+} & {
+    l: string | undefined;
+} & {
+    f: boolean;
+} & {
+    file: string | undefined;
+}, "file"> & {
+    file: string | undefined;
+}>;

package/lib/cli/commands/translate/builder.js

@@ -0,0 +1,5 @@
+import { commonOptions } from "../../common.js";
+export const builder = (yargs) => commonOptions(yargs).positional("file", {
+    describe: "Mulmo Script File",
+    type: "string",
+});

package/lib/cli/commands/translate/handler.d.ts

@@ -0,0 +1,4 @@
+import { CliArgs } from "../../../types/cli_types.js";
+export declare const handler: (argv: CliArgs<{
+    i?: string;
+}>) => Promise<void>;

package/lib/cli/commands/translate/handler.js

@@ -0,0 +1,6 @@
+import { translate } from "../../../actions/index.js";
+import { initializeContext } from "../../../cli/helpers.js";
+export const handler = async (argv) => {
+    const context = await initializeContext(argv);
+    await translate(context);
+};

package/lib/cli/commands/translate/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "translate <file>";
+export declare const desc = "Translate Mulmo script";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/translate/index.js

@@ -0,0 +1,4 @@
+export const command = "translate <file>";
+export const desc = "Translate Mulmo script";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/common.d.ts

@@ -1,8 +1,12 @@
 import type { Argv } from "yargs";
 export declare const commonOptions: (yargs: Argv) => Argv<{
-    v: boolean;
-} & {
     o: string | undefined;
 } & {
     b: string | undefined;
+} & {
+    l: string | undefined;
+} & {
+    f: boolean;
+} & {
+    file: string | undefined;
 }>;

package/lib/cli/common.js

@@ -1,12 +1,6 @@
+import { languages } from "../utils/const.js";
 export const commonOptions = (yargs) => {
     return yargs
-        .option("v", {
-        alias: "verbose",
-        describe: "verbose log",
-        demandOption: true,
-        default: false,
-        type: "boolean",
-    })
         .option("o", {
         alias: "outdir",
         description: "output dir",
@@ -18,5 +12,22 @@ export const commonOptions = (yargs) => {
         description: "base dir",
         demandOption: false,
         type: "string",
+    })
+        .option("l", {
+        alias: "lang",
+        description: "target language",
+        choices: languages,
+        demandOption: false,
+        type: "string",
+    })
+        .option("f", {
+        alias: "force",
+        describe: "Force regenerate",
+        type: "boolean",
+        default: false,
+    })
+        .positional("file", {
+        describe: "Mulmo Script File",
+        type: "string",
     });
 };

package/lib/cli/helpers.d.ts

@@ -0,0 +1,38 @@
+import type { MulmoStudioContext } from "../types/type.js";
+import type { CliArgs } from "../types/cli_types.js";
+export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
+export interface FileObject {
+    baseDirPath: string;
+    mulmoFilePath: string;
+    mulmoFileDirPath: string;
+    outDirPath: string;
+    imageDirPath: string;
+    audioDirPath: string;
+    isHttpPath: boolean;
+    fileOrUrl: string;
+    outputStudioFilePath: string;
+    fileName: string;
+}
+export declare const getFileObject: (args: {
+    basedir?: string;
+    outdir?: string;
+    imagedir?: string;
+    audiodir?: string;
+    file: string;
+}) => FileObject;
+export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<any>;
+type InitOptions = {
+    b?: string;
+    o?: string;
+    i?: string;
+    a?: string;
+    file?: string;
+    l?: string;
+    c?: string;
+};
+export declare const initializeContext: (argv: CliArgs<InitOptions>) => Promise<MulmoStudioContext>;
+export declare const runTranslateIfNeeded: (context: MulmoStudioContext, argv: {
+    l?: string;
+    c?: string;
+}) => Promise<void>;
+export {};

package/lib/cli/helpers.js

@@ -0,0 +1,115 @@
+import { GraphAILogger } from "graphai";
+import fs from "fs";
+import path from "path";
+import clipboardy from "clipboardy";
+import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath, resolveDirPath } from "../utils/file.js";
+import { isHttp } from "../utils/utils.js";
+import { createOrUpdateStudioData } from "../utils/preprocess.js";
+import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
+import { translate } from "../actions/translate.js";
+export const setGraphAILogger = (verbose, logValues) => {
+    if (verbose) {
+        if (logValues) {
+            Object.entries(logValues).forEach(([key, value]) => {
+                GraphAILogger.info(`${key}:`, value);
+            });
+        }
+    }
+    else {
+        GraphAILogger.setLevelEnabled("error", false);
+        GraphAILogger.setLevelEnabled("log", false);
+        GraphAILogger.setLevelEnabled("warn", false);
+    }
+};
+export const getFileObject = (args) => {
+    const { basedir, outdir, imagedir, audiodir, file } = args;
+    const baseDirPath = getBaseDirPath(basedir);
+    const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
+    const { fileOrUrl, fileName } = (() => {
+        if (file === "__clipboard") {
+            // We generate a new unique script file from clipboard text in the output directory
+            const now = new Date();
+            const pad = (n) => n.toString().padStart(2, "0");
+            const fileName = `script_${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+            const clipboardText = clipboardy.readSync();
+            const fileOrUrl = resolveDirPath(outDirPath, `${fileName}.json`);
+            fs.writeFileSync(fileOrUrl, clipboardText, "utf8");
+            return { fileOrUrl, fileName };
+        }
+        const fileOrUrl = file ?? "";
+        const fileName = path.parse(fileOrUrl).name;
+        return { fileOrUrl, fileName };
+    })();
+    const isHttpPath = isHttp(fileOrUrl);
+    const mulmoFilePath = isHttpPath ? "" : getFullPath(baseDirPath, fileOrUrl);
+    const mulmoFileDirPath = path.dirname(isHttpPath ? baseDirPath : mulmoFilePath);
+    const imageDirPath = getFullPath(outDirPath, imagedir ?? imageDirName);
+    const audioDirPath = getFullPath(outDirPath, audiodir ?? audioDirName);
+    const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
+    return {
+        baseDirPath,
+        mulmoFilePath,
+        mulmoFileDirPath,
+        outDirPath,
+        imageDirPath,
+        audioDirPath,
+        isHttpPath,
+        fileOrUrl,
+        outputStudioFilePath,
+        fileName,
+    };
+};
+export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
+    if (isHttpPath) {
+        const res = await fetchMulmoScriptFile(fileOrUrl);
+        if (!res.result || !res.script) {
+            GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
+            process.exit(1);
+        }
+        return res.script;
+    }
+    if (!fs.existsSync(mulmoFilePath)) {
+        GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
+        process.exit(1);
+    }
+    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath).mulmoData;
+};
+export const initializeContext = async (argv) => {
+    const files = getFileObject({
+        basedir: argv.b,
+        outdir: argv.o,
+        imagedir: argv.i,
+        audiodir: argv.a,
+        file: argv.file ?? "",
+    });
+    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath } = files;
+    setGraphAILogger(argv.v, {
+        files,
+    });
+    const mulmoScript = await fetchScript(isHttpPath, mulmoFilePath, fileOrUrl);
+    // Create or update MulmoStudio file with MulmoScript
+    const currentStudio = readMulmoScriptFile(outputStudioFilePath);
+    const studio = (() => {
+        try {
+            // validate mulmoStudioSchema. skip if __test_invalid__ is true
+            return createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
+        }
+        catch (error) {
+            GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
+            process.exit(1);
+        }
+    })();
+    return {
+        studio,
+        fileDirs: files,
+        force: Boolean(argv.f),
+        lang: argv.l,
+        caption: argv.c,
+    };
+};
+export const runTranslateIfNeeded = async (context, argv) => {
+    if (argv.l || argv.c) {
+        GraphAILogger.log("run translate");
+        await translate(context);
+    }
+};

package/lib/cli/tool-args.d.ts

@@ -3,6 +3,7 @@ export declare const args: {
     v: boolean;
     o: string | undefined;
     b: string | undefined;
+    l: string | undefined;
     u: string[] | never[];
     i: boolean | undefined;
     t: string | undefined;

package/lib/cli/tool-args.js

@@ -54,7 +54,7 @@ export const args = commonOptions(yargs(hideBin(process.argv)))
     .command("$0 <action>", "Run mulmocast tool", (yargs) => {
     return yargs.positional("action", {
         describe: "action to perform",
-        choices: ["scripting", "prompt"],
+        choices: ["scripting", "prompt", "schema"],
         type: "string",
     });
 })

package/lib/cli/tool-cli.js

@@ -8,6 +8,8 @@ import { getBaseDirPath, getFullPath } from "../utils/file.js";
 import { createMulmoScriptInteractively } from "../tools/create_mulmo_script_interactively.js";
 import { dumpPromptFromTemplate } from "../tools/dump_prompt.js";
 import { getUrlsIfNeeded, selectTemplate } from "../utils/inquirer.js";
+import { mulmoScriptSchema } from "../types/schema.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
 const main = async () => {
     const { o: outdir, b: basedir, action, v: verbose, i: interactive, s: filename, cache } = args;
     let { t: template } = args;
@@ -51,6 +53,12 @@ const main = async () => {
     else if (action === "prompt") {
         await dumpPromptFromTemplate({ templateName: template });
     }
+    else if (action === "schema") {
+        const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
+            strictUnions: true,
+        });
+        GraphAILogger.info(JSON.stringify(defaultSchema, null, 2));
+    }
     else {
         throw new Error(`Unknown or unsupported action: ${action}`);
     }

package/lib/methods/mulmo_script.d.ts

@@ -1,7 +1,6 @@
 import "dotenv/config";
 import { MulmoCanvasDimension, MulmoScript, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType } from "../types/index.js";
 export declare const MulmoScriptMethods: {
-    getPadding(script: MulmoScript): number;
     getCanvasSize(script: MulmoScript): MulmoCanvasDimension;
     getAspectRatio(script: MulmoScript): string;
     getSpeechProvider(script: MulmoScript): Text2SpeechProvider;

package/lib/methods/mulmo_script.js

@@ -3,8 +3,8 @@ import { text2ImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensi
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
-    "h1 { font-size: 56px; margin-bottom: 20px; text-align: center; }",
-    "h2 { font-size: 48px }",
+    "h1 { font-size: 56px; margin-bottom: 20px; text-align: center }",
+    "h2 { font-size: 48px; text-align: center }",
     "h3 { font-size: 36px }",
     "ul { margin-left: 40px } ",
     "pre { background: #eeeecc; font-size: 16px; padding:4px }",
@@ -15,9 +15,6 @@ const defaultTextSlideStyles = [
     "tr:nth-child(even) { background-color: #eee }",
 ];
 export const MulmoScriptMethods = {
-    getPadding(script) {
-        return script.videoParams?.padding ?? 1000; // msec
-    },
     getCanvasSize(script) {
         return mulmoCanvasDimensionSchema.parse(script.canvasSize);
     },
@@ -30,11 +27,11 @@ export const MulmoScriptMethods = {
         return text2SpeechProviderSchema.parse(script.speechParams?.provider);
     },
     getTextSlideStyle(script, beat) {
-        const styles = script.textSlideParams?.cssStyles ?? defaultTextSlideStyles;
+        const styles = script.textSlideParams?.cssStyles ?? [];
         // NOTES: Taking advantage of CSS override rule (you can redefine it to override)
         const extraStyles = beat.textSlideParams?.cssStyles ?? [];
         // This code allows us to support both string and array of strings for cssStyles
-        return [...[styles], ...[extraStyles]].flat().join("\n");
+        return [...defaultTextSlideStyles, ...[styles], ...[extraStyles]].flat().join("\n");
     },
     getSpeechOptions(script, beat) {
         return { ...script.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };

package/lib/methods/mulmo_script_template.d.ts

@@ -1,4 +1,4 @@
-import { MulmoScriptTemplate } from "../types/index.js";
+import { MulmoScript, MulmoScriptTemplate } from "../types/index.js";
 export declare const MulmoScriptTemplateMethods: {
-    getSystemPrompt(template: MulmoScriptTemplate): string;
+    getSystemPrompt(template: MulmoScriptTemplate, script?: MulmoScript): string;
 };

package/lib/methods/mulmo_script_template.js

@@ -1,16 +1,6 @@
-import { zodToJsonSchema } from "zod-to-json-schema";
-import { mulmoScriptSchema } from "../types/schema.js";
+import { getMulmoScriptTemplateSystemPrompt } from "../utils/prompt.js";
 export const MulmoScriptTemplateMethods = {
-    getSystemPrompt(template) {
-        // script is provided, use it as a script template
-        if (template.script) {
-            return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(template.script)}\n\`\`\``;
-        }
-        // script is not provided, use the default schema
-        const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
-            strictUnions: true,
-        });
-        const specificOutputPrompt = `The output should follow the JSON schema specified below. Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`;
-        return `${template.systemPrompt}\n\n${specificOutputPrompt}\n\n\`\`\`JSON\n${JSON.stringify(defaultSchema)}\n\`\`\``;
+    getSystemPrompt(template, script) {
+        return getMulmoScriptTemplateSystemPrompt(template, script);
     },
 };

package/lib/methods/mulmo_studio.d.ts

@@ -0,0 +1,8 @@
+import { MulmoStudio } from "../types/index.js";
+type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
+type BeatSessionType = "audio" | "image" | "multiLingual" | "caption";
+export declare const MulmoStudioMethods: {
+    setSessionState(studio: MulmoStudio, sessionType: SessionType, value: boolean): void;
+    setBeatSessionState(studio: MulmoStudio, sessionType: BeatSessionType, index: number, value: boolean): void;
+};
+export {};

package/lib/methods/mulmo_studio.js

@@ -0,0 +1,24 @@
+import { GraphAILogger } from "graphai";
+const notifyStateChange = (studio, sessionType) => {
+    const prefix = studio.state.inSession[sessionType] ? "<" : " >";
+    GraphAILogger.info(`${prefix} ${sessionType}`);
+};
+const notifyBeatStateChange = (studio, sessionType, index) => {
+    const prefix = studio.state.inBeatSession[sessionType].has(index) ? "{" : " }";
+    GraphAILogger.info(`${prefix} ${sessionType} ${index}`);
+};
+export const MulmoStudioMethods = {
+    setSessionState(studio, sessionType, value) {
+        studio.state.inSession[sessionType] = value;
+        notifyStateChange(studio, sessionType);
+    },
+    setBeatSessionState(studio, sessionType, index, value) {
+        if (value) {
+            studio.state.inBeatSession[sessionType].add(index);
+        }
+        else {
+            studio.state.inBeatSession[sessionType].delete(index);
+        }
+        notifyBeatStateChange(studio, sessionType, index);
+    },
+};

package/lib/tools/create_mulmo_script_from_url.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
-export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath, llm, llm_model }: ScriptingParams) => Promise<void>;