mulmocast 0.0.14 → 0.0.16

package/lib/agents/tavily_agent.js

@@ -0,0 +1,130 @@
+import { assert } from "graphai";
+import { tavily } from "@tavily/core";
+const getTavilyApiKey = (params, config) => {
+    if (params?.apiKey) {
+        return params.apiKey;
+    }
+    if (config?.apiKey) {
+        return config.apiKey;
+    }
+    return typeof process !== "undefined" ? process?.env?.TAVILY_API_KEY : null;
+};
+export const tavilySearchAgent = async ({ namedInputs, params, config, }) => {
+    const { query } = namedInputs;
+    assert(!!query, "tavilySearchAgent: query is required! set inputs: { query: 'search terms' }");
+    try {
+        const apiKey = getTavilyApiKey(params, config);
+        assert(apiKey, "Tavily API key is required. Please set the TAVILY_API_KEY environment variable or provide it in params/config.");
+        const tvly = tavily({ apiKey });
+        // Convert params to SDK options format
+        const sdkOptions = {};
+        if (params?.max_results !== undefined)
+            sdkOptions.maxResults = params.max_results;
+        if (params?.search_depth !== undefined)
+            sdkOptions.searchDepth = params.search_depth;
+        if (params?.include_answer !== undefined)
+            sdkOptions.includeAnswer = params.include_answer;
+        if (params?.include_raw_content !== undefined)
+            sdkOptions.includeRawContent = params.include_raw_content;
+        const response = await tvly.search(query, sdkOptions);
+        return response;
+    }
+    catch (error) {
+        throw new Error(`Tavily search failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+};
+const tavilySearchAgentInfo = {
+    name: "tavilySearchAgent",
+    agent: tavilySearchAgent,
+    mock: tavilySearchAgent,
+    params: {
+        type: "object",
+        properties: {
+            apiKey: {
+                type: "string",
+                description: "Tavily API key",
+            },
+            max_results: {
+                type: "number",
+                description: "Maximum number of search results to return (default: 5)",
+            },
+            search_depth: {
+                type: "string",
+                enum: ["basic", "advanced"],
+                description: "Search depth - basic for faster results, advanced for more comprehensive results",
+            },
+            include_answer: {
+                type: "boolean",
+                description: "Include a direct answer to the query when available",
+            },
+            include_raw_content: {
+                type: "string",
+                enum: ["boolean", "markdown", "text"],
+                description: "Include raw content from search results (boolean, markdown, text)",
+            },
+        },
+    },
+    inputs: {
+        type: "object",
+        properties: {
+            query: {
+                type: "string",
+                description: "Search query string",
+            },
+        },
+        required: ["query"],
+    },
+    output: {
+        type: "object",
+        properties: {
+            query: { type: "string" },
+            answer: { type: ["string", "null"] },
+            results: {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        title: { type: "string" },
+                        url: { type: "string" },
+                        content: { type: "string" },
+                        rawContent: { type: ["string", "null"] },
+                        score: { type: "number" },
+                    },
+                },
+            },
+        },
+    },
+    samples: [
+        {
+            inputs: {
+                query: "latest AI developments 2024",
+            },
+            params: {
+                max_results: 3,
+                include_answer: true,
+            },
+            result: {
+                query: "latest AI developments 2024",
+                answer: "Recent AI developments in 2024 include...",
+                results: [
+                    {
+                        title: "Major AI Breakthroughs in 2024",
+                        url: "https://example.com/ai-2024",
+                        content: "The year 2024 has seen significant advances in artificial intelligence...",
+                        rawContent: null,
+                        score: 0.95,
+                    },
+                ],
+            },
+        },
+    ],
+    description: "Performs web search using Tavily API and returns relevant search results with optional AI-generated answers",
+    category: ["search", "web"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/tree/main/src/agents/tavily_agent.ts",
+    source: "https://github.com/receptron/mulmocast-cli/tree/main/src/agents/tavily_agent.ts",
+    package: "@receptron/mulmocast-cli",
+    license: "MIT",
+    environmentVariables: ["TAVILY_API_KEY"],
+};
+export default tavilySearchAgentInfo;

package/lib/agents/tts_openai_agent.js

@@ -24,7 +24,15 @@ export const ttsOpenaiAgent = async ({ namedInputs, params }) => {
                 error: e,
             };
         }
-        GraphAILogger.info(e);
+        GraphAILogger.error(e);
+        if (e && typeof e === "object" && "error" in e) {
+            GraphAILogger.info("tts_openai_agent: ");
+            GraphAILogger.info(e.error);
+        }
+        else if (e instanceof Error) {
+            GraphAILogger.info("tts_openai_agent: ");
+            GraphAILogger.info(e.message);
+        }
         throw new Error("TTS OpenAI Error");
     }
 };

package/lib/cli/commands/audio/builder.d.ts

@@ -7,6 +7,10 @@ export declare const builder: (yargs: Argv) => Argv<{
     l: string | undefined;
 } & {
     f: boolean;
+} & {
+    dryRun: boolean;
+} & {
+    p: string | undefined;
 } & {
     file: string | undefined;
 } & {

package/lib/cli/commands/image/builder.d.ts

@@ -7,6 +7,10 @@ export declare const builder: (yargs: Argv) => Argv<{
     l: string | undefined;
 } & {
     f: boolean;
+} & {
+    dryRun: boolean;
+} & {
+    p: string | undefined;
 } & {
     file: string | undefined;
 } & {

package/lib/cli/commands/movie/builder.d.ts

@@ -7,6 +7,10 @@ export declare const builder: (yargs: Argv) => Argv<{
     l: string | undefined;
 } & {
     f: boolean;
+} & {
+    dryRun: boolean;
+} & {
+    p: string | undefined;
 } & {
     file: string | undefined;
 } & {

package/lib/cli/commands/pdf/builder.d.ts

@@ -7,6 +7,10 @@ export declare const builder: (yargs: Argv) => Argv<{
     l: string | undefined;
 } & {
     f: boolean;
+} & {
+    dryRun: boolean;
+} & {
+    p: string | undefined;
 } & {
     file: string | undefined;
 } & {

package/lib/cli/commands/translate/builder.d.ts

@@ -7,6 +7,10 @@ export declare const builder: (yargs: Argv) => Argv<import("yargs").Omit<{
     l: string | undefined;
 } & {
     f: boolean;
+} & {
+    dryRun: boolean;
+} & {
+    p: string | undefined;
 } & {
     file: string | undefined;
 }, "file"> & {

package/lib/cli/common.d.ts

@@ -7,6 +7,10 @@ export declare const commonOptions: (yargs: Argv) => Argv<{
     l: string | undefined;
 } & {
     f: boolean;
+} & {
+    dryRun: boolean;
+} & {
+    p: string | undefined;
 } & {
     file: string | undefined;
 }>;

package/lib/cli/common.js

@@ -25,6 +25,17 @@ export const commonOptions = (yargs) => {
         describe: "Force regenerate",
         type: "boolean",
         default: false,
+    })
+        .option("dryRun", {
+        describe: "Dry run",
+        type: "boolean",
+        default: false,
+    })
+        .option("p", {
+        alias: "presentationStyle",
+        describe: "Presentation Style",
+        demandOption: false,
+        type: "string",
     })
         .positional("file", {
         describe: "Mulmo Script File",

package/lib/cli/helpers.d.ts

@@ -1,4 +1,4 @@
-import type { MulmoScript, MulmoStudioContext } from "../types/type.js";
+import type { MulmoScript, MulmoStudioContext, MulmoPresentationStyle } from "../types/type.js";
 import type { CliArgs } from "../types/cli_types.js";
 export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
 export interface FileObject {
@@ -11,6 +11,7 @@ export interface FileObject {
     isHttpPath: boolean;
     fileOrUrl: string;
     outputStudioFilePath: string;
+    presentationStylePath: string | undefined;
     fileName: string;
 }
 export declare const getFileObject: (args: {
@@ -18,9 +19,11 @@ export declare const getFileObject: (args: {
     outdir?: string;
     imagedir?: string;
     audiodir?: string;
+    presentationStyle?: string;
     file: string;
 }) => FileObject;
 export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
+export declare const getPresentationStyle: (presentationStylePath: string | undefined) => MulmoPresentationStyle | null;
 type InitOptions = {
     b?: string;
     o?: string;
@@ -29,6 +32,7 @@ type InitOptions = {
     file?: string;
     l?: string;
     c?: string;
+    p?: string;
 };
 export declare const initializeContext: (argv: CliArgs<InitOptions>) => Promise<MulmoStudioContext | null>;
 export declare const runTranslateIfNeeded: (context: MulmoStudioContext, argv: {

package/lib/cli/helpers.js

@@ -7,6 +7,7 @@ import { isHttp } from "../utils/utils.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
 import { translate } from "../actions/translate.js";
+import { mulmoPresentationStyleSchema } from "../types/schema.js";
 export const setGraphAILogger = (verbose, logValues) => {
     if (verbose) {
         if (logValues) {
@@ -22,7 +23,7 @@ export const setGraphAILogger = (verbose, logValues) => {
     }
 };
 export const getFileObject = (args) => {
-    const { basedir, outdir, imagedir, audiodir, file } = args;
+    const { basedir, outdir, imagedir, audiodir, file, presentationStyle } = args;
     const baseDirPath = getBaseDirPath(basedir);
     const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
     const { fileOrUrl, fileName } = (() => {
@@ -47,6 +48,7 @@ export const getFileObject = (args) => {
     const imageDirPath = getFullPath(outDirPath, imagedir ?? imageDirName);
     const audioDirPath = getFullPath(outDirPath, audiodir ?? audioDirName);
     const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
+    const presentationStylePath = presentationStyle ? getFullPath(baseDirPath, presentationStyle) : undefined;
     return {
         baseDirPath,
         mulmoFilePath,
@@ -57,6 +59,7 @@ export const getFileObject = (args) => {
         isHttpPath,
         fileOrUrl,
         outputStudioFilePath,
+        presentationStylePath,
         fileName,
     };
 };
@@ -75,15 +78,26 @@ export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
     }
     return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath)?.mulmoData ?? null;
 };
+export const getPresentationStyle = (presentationStylePath) => {
+    if (presentationStylePath) {
+        if (!fs.existsSync(presentationStylePath)) {
+            throw new Error(`ERROR: File not exists ${presentationStylePath}`);
+        }
+        const jsonData = readMulmoScriptFile(presentationStylePath, "ERROR: File does not exist " + presentationStylePath)?.mulmoData ?? null;
+        return mulmoPresentationStyleSchema.parse(jsonData);
+    }
+    return null;
+};
 export const initializeContext = async (argv) => {
     const files = getFileObject({
         basedir: argv.b,
         outdir: argv.o,
         imagedir: argv.i,
         audiodir: argv.a,
+        presentationStyle: argv.p,
         file: argv.file ?? "",
     });
-    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath } = files;
+    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath, presentationStylePath } = files;
     setGraphAILogger(argv.v, {
         files,
     });
@@ -91,6 +105,7 @@ export const initializeContext = async (argv) => {
     if (!mulmoScript) {
         return null;
     }
+    const presentationStyle = getPresentationStyle(presentationStylePath);
     // Create or update MulmoStudio file with MulmoScript
     const currentStudio = readMulmoScriptFile(outputStudioFilePath);
     try {
@@ -100,6 +115,7 @@ export const initializeContext = async (argv) => {
             studio,
             fileDirs: files,
             force: Boolean(argv.f),
+            dryRun: Boolean(argv.dryRun),
             lang: argv.l,
             caption: argv.c,
             sessionState: {
@@ -119,6 +135,7 @@ export const initializeContext = async (argv) => {
                     caption: {},
                 },
             },
+            presentationStyle: presentationStyle ?? studio.script,
         };
     }
     catch (error) {

package/lib/methods/index.d.ts

@@ -1,4 +1,4 @@
-export * from "./mulmo_script.js";
+export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";

package/lib/methods/index.js

@@ -1,4 +1,4 @@
-export * from "./mulmo_script.js";
+export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";

package/lib/methods/mulmo_presentation_style.d.ts

@@ -0,0 +1,14 @@
+import "dotenv/config";
+import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData } from "../types/index.js";
+export declare const MulmoPresentationStyleMethods: {
+    getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
+    getSpeechProvider(presentationStyle: MulmoPresentationStyle): Text2SpeechProvider;
+    getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
+    getTextSlideStyle(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
+    getSpeechOptions(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeechOptions | undefined;
+    getSpeaker(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeakerData;
+    getProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
+    getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
+    getImageAgentInfo(presentationStyle: MulmoPresentationStyle, dryRun?: boolean): Text2ImageAgentInfo;
+    getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;
+};

package/lib/methods/mulmo_presentation_style.js

@@ -0,0 +1,70 @@
+import "dotenv/config";
+import { text2ImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
+const defaultTextSlideStyles = [
+    '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
+    "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
+    "h1 { font-size: 56px; margin-bottom: 20px; text-align: center }",
+    "h2 { font-size: 48px; text-align: center }",
+    "h3 { font-size: 36px }",
+    "ul { margin-left: 40px } ",
+    "pre { background: #eeeecc; font-size: 16px; padding:4px }",
+    "p { margin-left: 40px }",
+    "table { font-size: 36px; margin: auto; border: 1px solid gray; border-collapse: collapse }",
+    "th { border-bottom: 1px solid gray }",
+    "td, th { padding: 8px }",
+    "tr:nth-child(even) { background-color: #eee }",
+];
+export const MulmoPresentationStyleMethods = {
+    getCanvasSize(presentationStyle) {
+        return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
+    },
+    getSpeechProvider(presentationStyle) {
+        return text2SpeechProviderSchema.parse(presentationStyle.speechParams?.provider);
+    },
+    getAllSpeechProviders(presentationStyle) {
+        const providers = new Set();
+        const defaultProvider = this.getSpeechProvider(presentationStyle);
+        Object.values(presentationStyle.speechParams.speakers).forEach((speaker) => {
+            const provider = speaker.provider ?? defaultProvider;
+            providers.add(provider);
+        });
+        return providers;
+    },
+    getTextSlideStyle(presentationStyle, beat) {
+        const styles = presentationStyle.textSlideParams?.cssStyles ?? [];
+        // NOTES: Taking advantage of CSS override rule (you can redefine it to override)
+        const extraStyles = beat.textSlideParams?.cssStyles ?? [];
+        // This code allows us to support both string and array of strings for cssStyles
+        return [...defaultTextSlideStyles, ...[styles], ...[extraStyles]].flat().join("\n");
+    },
+    getSpeechOptions(presentationStyle, beat) {
+        return { ...presentationStyle.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
+    },
+    getSpeaker(presentationStyle, beat) {
+        return presentationStyle.speechParams.speakers[beat.speaker];
+    },
+    getProvider(presentationStyle, beat) {
+        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
+        return speaker.provider ?? presentationStyle.speechParams.provider;
+    },
+    getVoiceId(presentationStyle, beat) {
+        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
+        return speaker.voiceId;
+    },
+    getImageAgentInfo(presentationStyle, dryRun = false) {
+        // Notice that we copy imageParams from presentationStyle and update
+        // provider and model appropriately.
+        const provider = text2ImageProviderSchema.parse(presentationStyle.imageParams?.provider);
+        const defaultImageParams = {
+            model: provider === "openai" ? process.env.DEFAULT_OPENAI_IMAGE_MODEL : undefined,
+        };
+        return {
+            provider,
+            agent: dryRun ? "mediaMockAgent" : provider === "google" ? "imageGoogleAgent" : "imageOpenaiAgent",
+            imageParams: { ...defaultImageParams, ...presentationStyle.imageParams },
+        };
+    },
+    getImageType(_, beat) {
+        return beat.image?.type == "movie" ? "movie" : "image";
+    },
+};

package/lib/methods/mulmo_script.d.ts

@@ -6,6 +6,6 @@ export declare const MulmoScriptMethods: {
     getAllSpeechProviders(script: MulmoScript): Set<Text2SpeechProvider>;
     getTextSlideStyle(script: MulmoScript, beat: MulmoBeat): string;
     getSpeechOptions(script: MulmoScript, beat: MulmoBeat): SpeechOptions | undefined;
-    getImageAgentInfo(script: MulmoScript): Text2ImageAgentInfo;
+    getImageAgentInfo(script: MulmoScript, dryRun?: boolean): Text2ImageAgentInfo;
     getImageType(_: MulmoScript, beat: MulmoBeat): BeatMediaType;
 };

package/lib/methods/mulmo_script.js

@@ -40,7 +40,7 @@ export const MulmoScriptMethods = {
     getSpeechOptions(script, beat) {
         return { ...script.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
     },
-    getImageAgentInfo(script) {
+    getImageAgentInfo(script, dryRun = false) {
         // Notice that we copy imageParams from script and update
         // provider and model appropriately.
         const provider = text2ImageProviderSchema.parse(script.imageParams?.provider);
@@ -49,7 +49,7 @@ export const MulmoScriptMethods = {
         };
         return {
             provider,
-            agent: provider === "google" ? "imageGoogleAgent" : "imageOpenaiAgent",
+            agent: dryRun ? "mediaMockAgent" : provider === "google" ? "imageGoogleAgent" : "imageOpenaiAgent",
             imageParams: { ...defaultImageParams, ...script.imageParams },
         };
     },

package/lib/methods/mulmo_studio_context.d.ts

@@ -1,8 +1,22 @@
 import { MulmoStudioContext } from "../types/index.js";
 type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
 type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
+type SessionProgressEvent = {
+    kind: "session";
+    sessionType: SessionType;
+    inSession: boolean;
+} | {
+    kind: "beat";
+    sessionType: BeatSessionType;
+    index: number;
+    inSession: boolean;
+};
+type SessionProgressCallback = (change: SessionProgressEvent) => void;
+export declare const addSessionProgressCallback: (cb: SessionProgressCallback) => void;
+export declare const removeSessionProgressCallback: (cb: SessionProgressCallback) => void;
 export declare const MulmoStudioContextMethods: {
     resolveAssetPath(context: MulmoStudioContext, relativePath: string): string;
+    getAudioDirPath(context: MulmoStudioContext): string;
     setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
 };

package/lib/methods/mulmo_studio_context.js

@@ -1,17 +1,35 @@
 import path from "path";
 import { GraphAILogger } from "graphai";
+const sessionProgressCallbacks = new Set();
+export const addSessionProgressCallback = (cb) => {
+    sessionProgressCallbacks.add(cb);
+};
+export const removeSessionProgressCallback = (cb) => {
+    sessionProgressCallbacks.delete(cb);
+};
 const notifyStateChange = (context, sessionType) => {
-    const prefix = context.sessionState.inSession[sessionType] ? "<" : " >";
+    const inSession = context.sessionState.inSession[sessionType] ?? false;
+    const prefix = inSession ? "<" : " >";
     GraphAILogger.info(`${prefix} ${sessionType}`);
+    for (const callback of sessionProgressCallbacks) {
+        callback({ kind: "session", sessionType, inSession });
+    }
 };
 const notifyBeatStateChange = (context, sessionType, index) => {
-    const prefix = context.sessionState.inBeatSession[sessionType][index] ? "{" : " }";
+    const inSession = context.sessionState.inBeatSession[sessionType][index] ?? false;
+    const prefix = inSession ? "{" : " }";
     GraphAILogger.info(`${prefix} ${sessionType} ${index}`);
+    for (const callback of sessionProgressCallbacks) {
+        callback({ kind: "beat", sessionType, index, inSession });
+    }
 };
 export const MulmoStudioContextMethods = {
     resolveAssetPath(context, relativePath) {
         return path.resolve(context.fileDirs.mulmoFileDirPath, relativePath);
     },
+    getAudioDirPath(context) {
+        return context.fileDirs.audioDirPath;
+    },
     setSessionState(context, sessionType, value) {
         context.sessionState.inSession[sessionType] = value;
         notifyStateChange(context, sessionType);

package/lib/tools/deep_research.d.ts

@@ -0,0 +1,2 @@
+import "dotenv/config";
+export declare const deepResearch: () => Promise<void>;