mulmocast 0.0.1

package/lib/types/schema.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.urlsSchema = exports.mulmoScriptTemplateSchema = exports.mulmoStudioSchema = exports.mulmoStudioBeatSchema = exports.mulmoScriptSchema = exports.mulmoSpeechParamsSchema = exports.mulmoCastCreditSchema = exports.mulmoDimensionSchema = exports.mulmoBeatSchema = exports.videoParamsSchema = exports.textSlideParamsSchema = exports.mulmoImageParamsSchema = exports.mulmoAudioAssetSchema = exports.mulmoMediaSchema = exports.speakerDictionarySchema = exports.speechOptionsSchema = exports.multiLingualTextsSchema = exports.localizedTextSchema = exports.langSchema = void 0;
+const zod_1 = require("zod");
+exports.langSchema = zod_1.z.string();
+const URLStringSchema = zod_1.z.string().url();
+exports.localizedTextSchema = zod_1.z
+    .object({
+    text: zod_1.z.string(),
+    lang: zod_1.z.string(),
+    // caption: z.string(),
+    texts: zod_1.z.array(zod_1.z.string()).optional(),
+    ttsTexts: zod_1.z.array(zod_1.z.string()).optional(),
+    duration: zod_1.z.number().optional(), // generated // video duration time(ms)
+    // filename: z.string().optional(), // generated //
+})
+    .strict();
+exports.multiLingualTextsSchema = zod_1.z.record(exports.langSchema, exports.localizedTextSchema);
+exports.speechOptionsSchema = zod_1.z
+    .object({
+    speed: zod_1.z.number().optional(), // default: 1.0
+    instruction: zod_1.z.string().optional(),
+})
+    .strict();
+const speakerIdSchema = zod_1.z.string();
+const speakerDataSchema = zod_1.z
+    .object({
+    displayName: zod_1.z.record(exports.langSchema, zod_1.z.string()),
+    voiceId: zod_1.z.string(),
+    speechOptions: exports.speechOptionsSchema.optional(),
+})
+    .strict();
+exports.speakerDictionarySchema = zod_1.z.record(speakerIdSchema, speakerDataSchema);
+const mediaSourceSchema = zod_1.z.discriminatedUnion("kind", [
+    zod_1.z.object({ kind: zod_1.z.literal("url"), url: URLStringSchema }).strict(), // https://example.com/foo.pdf
+    zod_1.z.object({ kind: zod_1.z.literal("data"), data: zod_1.z.string() }).strict(), // base64
+    zod_1.z.object({ kind: zod_1.z.literal("path"), path: zod_1.z.string() }).strict(), // foo.pdf
+]);
+// String is easier for AI, string array is easier for human
+const stringOrStringArray = zod_1.z.union([zod_1.z.string(), zod_1.z.array(zod_1.z.string())]);
+const MulmoMarkdownMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("markdown"),
+    markdown: stringOrStringArray,
+})
+    .strict();
+const MulmoWebMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("web"),
+    url: URLStringSchema,
+})
+    .strict();
+const MulmoPdfMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("pdf"),
+    source: mediaSourceSchema,
+})
+    .strict();
+const MulmoImageMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("image"),
+    source: mediaSourceSchema,
+})
+    .strict();
+const MulmoSvgMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("svg"),
+    source: mediaSourceSchema,
+})
+    .strict();
+const MulmoMovieMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("movie"),
+    source: mediaSourceSchema,
+})
+    .strict();
+const MulmoTextSlideMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("textSlide"),
+    slide: zod_1.z.object({
+        title: zod_1.z.string(),
+        bullets: zod_1.z.array(zod_1.z.string()),
+    }),
+})
+    .strict();
+exports.mulmoMediaSchema = zod_1.z.union([
+    MulmoMarkdownMediaSchema,
+    MulmoWebMediaSchema,
+    MulmoPdfMediaSchema,
+    MulmoImageMediaSchema,
+    MulmoSvgMediaSchema,
+    MulmoMovieMediaSchema,
+    MulmoTextSlideMediaSchema,
+]);
+const MulmoAudioMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("audio"),
+    source: mediaSourceSchema,
+})
+    .strict();
+const MulmoMidiMediaSchema = zod_1.z
+    .object({
+    type: zod_1.z.literal("midi"),
+    source: zod_1.z.string(), // TODO: define it later
+})
+    .strict();
+exports.mulmoAudioAssetSchema = zod_1.z.union([MulmoAudioMediaSchema, MulmoMidiMediaSchema]);
+exports.mulmoImageParamsSchema = zod_1.z
+    .object({
+    model: zod_1.z.string().optional(), // default: provider specific
+    size: zod_1.z.string().optional(), // default: provider specific
+    style: zod_1.z.string().optional(), // optional image style
+    moderation: zod_1.z.string().optional(), // optional image style
+})
+    .strict();
+exports.textSlideParamsSchema = zod_1.z
+    .object({
+    cssStyles: zod_1.z.array(zod_1.z.string()),
+})
+    .strict();
+exports.videoParamsSchema = zod_1.z
+    .object({
+    padding: zod_1.z.number().optional(), // msec
+})
+    .strict();
+exports.mulmoBeatSchema = zod_1.z
+    .object({
+    speaker: speakerIdSchema,
+    text: zod_1.z.string(),
+    image: exports.mulmoMediaSchema.optional(),
+    audio: exports.mulmoAudioAssetSchema.optional(),
+    imageParams: exports.mulmoImageParamsSchema.optional(), // beat specific parameters
+    speechOptions: exports.speechOptionsSchema.optional(),
+    textSlideParams: exports.textSlideParamsSchema.optional(),
+    imagePrompt: zod_1.z.string().optional(), // specified or inserted by preprocessor
+})
+    .strict();
+exports.mulmoDimensionSchema = zod_1.z
+    .object({
+    width: zod_1.z.number(),
+    height: zod_1.z.number(),
+})
+    .strict();
+// export const voiceMapSchema = z.record(speakerIdSchema, z.string())
+exports.mulmoCastCreditSchema = zod_1.z
+    .object({
+    version: zod_1.z.literal("1.0"),
+    credit: zod_1.z.literal("closing").optional(),
+})
+    .strict();
+exports.mulmoSpeechParamsSchema = zod_1.z
+    .object({
+    provider: zod_1.z.string().optional(),
+    speakers: exports.speakerDictionarySchema,
+})
+    .strict();
+exports.mulmoScriptSchema = zod_1.z
+    .object({
+    // global settings
+    $mulmocast: exports.mulmoCastCreditSchema,
+    title: zod_1.z.string(),
+    description: zod_1.z.string().optional(),
+    reference: zod_1.z.string().optional(),
+    lang: exports.langSchema.optional(), // default "en"
+    canvasSize: exports.mulmoDimensionSchema.optional(),
+    beats: zod_1.z.array(exports.mulmoBeatSchema),
+    speechParams: exports.mulmoSpeechParamsSchema,
+    imageParams: exports.mulmoImageParamsSchema
+        .extend({
+        provider: zod_1.z.string().optional(),
+    })
+        .optional(),
+    // for textSlides
+    textSlideParams: exports.textSlideParamsSchema.optional(),
+    videoParams: exports.videoParamsSchema.optional(),
+    // images: ImageInfo[] // generated
+    imagePath: zod_1.z.string().optional(), // for keynote images movie ??
+    omitCaptions: zod_1.z.boolean().optional(), // default is false
+    // for debugging
+    __test_invalid__: zod_1.z.boolean().optional(),
+})
+    .strict();
+exports.mulmoStudioBeatSchema = exports.mulmoBeatSchema
+    .extend({
+    multiLingualTexts: exports.multiLingualTextsSchema.optional(),
+    hash: zod_1.z.string().optional(),
+    duration: zod_1.z.number().optional(),
+    audioFile: zod_1.z.string().optional(),
+    imageFile: zod_1.z.string().optional(), // path to the image
+})
+    .strict();
+exports.mulmoStudioSchema = zod_1.z
+    .object({
+    script: exports.mulmoScriptSchema,
+    filename: zod_1.z.string(),
+    beats: zod_1.z.array(exports.mulmoStudioBeatSchema),
+})
+    .strict();
+exports.mulmoScriptTemplateSchema = zod_1.z
+    .object({
+    title: zod_1.z.string(),
+    description: zod_1.z.string(),
+    systemPrompt: zod_1.z.string(),
+    script: exports.mulmoScriptSchema.optional(),
+})
+    .strict();
+exports.urlsSchema = zod_1.z.array(zod_1.z.string().url({ message: "Invalid URL format" }));

package/lib/types/type.d.ts

@@ -0,0 +1,33 @@
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoDimensionSchema, mulmoScriptTemplateSchema } from "./schema";
+import { z } from "zod";
+export type LANG = z.infer<typeof langSchema>;
+export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
+export type SpeakerDictonary = z.infer<typeof speakerDictionarySchema>;
+export type MulmoSpeechParams = z.infer<typeof mulmoSpeechParamsSchema>;
+export type SpeechOptions = z.infer<typeof speechOptionsSchema>;
+export type MulmoImageParams = z.infer<typeof mulmoImageParamsSchema>;
+export type TextSlideParams = z.infer<typeof textSlideParamsSchema>;
+export type LocalizedText = z.infer<typeof localizedTextSchema>;
+export type MulmoScript = z.infer<typeof mulmoScriptSchema>;
+export type MulmoDimension = z.infer<typeof mulmoDimensionSchema>;
+export type MulmoStudioBeat = z.infer<typeof mulmoStudioBeatSchema>;
+export type MulmoStudio = z.infer<typeof mulmoStudioSchema>;
+export type MulmoScriptTemplate = z.infer<typeof mulmoScriptTemplateSchema>;
+export type FileDirs = {
+    mulmoFilePath: string;
+    mulmoFileDirPath: string;
+    baseDirPath: string;
+    outDirPath: string;
+    imageDirPath: string;
+    scratchpadDirPath: string;
+};
+export type MulmoStudioContext = {
+    fileDirs: FileDirs;
+    studio: MulmoStudio;
+};
+export type ScriptingParams = {
+    urls: string[];
+    outDirPath: string;
+    templateName: string;
+    filename: string;
+};

package/lib/types/type.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/utils/const.d.ts

@@ -0,0 +1,3 @@
+export declare const outDirName = "output";
+export declare const scratchpadDirName = "scratchpad";
+export declare const imageDirName = "images";

package/lib/utils/const.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.imageDirName = exports.scratchpadDirName = exports.outDirName = void 0;
+exports.outDirName = "output";
+exports.scratchpadDirName = "scratchpad";
+exports.imageDirName = "images";

package/lib/utils/file.d.ts

@@ -0,0 +1,28 @@
+import { MulmoScript, MulmoScriptTemplate } from "../types";
+export declare function readMulmoScriptFile<T = MulmoScript>(path: string, errorMessage: string): {
+    mulmoData: T;
+    mulmoDataPath: string;
+    fileName: string;
+};
+export declare function readMulmoScriptFile<T = MulmoScript>(path: string): {
+    mulmoData: T;
+    mulmoDataPath: string;
+    fileName: string;
+} | null;
+export declare const getOutputStudioFilePath: (outDirPath: string, fileName: string) => string;
+export declare const getOutputBGMFilePath: (outDirPath: string, fileName: string) => string;
+export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string) => string;
+export declare const getOutputAudioFilePath: (outDirPath: string, fileName: string) => string;
+export declare const getScratchpadFilePath: (scratchpadDirName: string, fileName: string) => string;
+export declare const getTemplateFilePath: (templateName: string) => string;
+export declare const mkdir: (dirPath: string) => void;
+export declare const silentPath: string;
+export declare const silentLastPath: string;
+export declare const defaultBGMPath: string;
+export declare const getBaseDirPath: (basedir?: string) => string;
+export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
+export declare const readTemplatePrompt: (templateName: string) => string;
+export declare const getAvailableTemplates: () => (MulmoScriptTemplate & {
+    filename: string;
+})[];
+export declare const writingMessage: (filePath: string) => void;

package/lib/utils/file.js

@@ -0,0 +1,112 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writingMessage = exports.getAvailableTemplates = exports.readTemplatePrompt = exports.getFullPath = exports.getBaseDirPath = exports.defaultBGMPath = exports.silentLastPath = exports.silentPath = exports.mkdir = exports.getTemplateFilePath = exports.getScratchpadFilePath = exports.getOutputAudioFilePath = exports.getOutputVideoFilePath = exports.getOutputBGMFilePath = exports.getOutputStudioFilePath = void 0;
+exports.readMulmoScriptFile = readMulmoScriptFile;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const graphai_1 = require("graphai");
+const mulmo_script_template_1 = require("../methods/mulmo_script_template");
+const schema_1 = require("../types/schema");
+function readMulmoScriptFile(arg2, errorMessage) {
+    const scriptPath = path_1.default.resolve(arg2);
+    if (!fs_1.default.existsSync(scriptPath)) {
+        if (errorMessage) {
+            console.error(errorMessage);
+            process.exit(1);
+        }
+        return null;
+    }
+    const scriptData = fs_1.default.readFileSync(scriptPath, "utf-8");
+    const script = JSON.parse(scriptData);
+    const parsedPath = path_1.default.parse(scriptPath);
+    return {
+        mulmoData: script,
+        mulmoDataPath: scriptPath,
+        fileName: parsedPath.name,
+    };
+}
+const getOutputStudioFilePath = (outDirPath, fileName) => {
+    return path_1.default.resolve(outDirPath, fileName + "_studio.json");
+};
+exports.getOutputStudioFilePath = getOutputStudioFilePath;
+const getOutputBGMFilePath = (outDirPath, fileName) => {
+    return path_1.default.resolve(outDirPath, fileName + "_bgm.mp3");
+};
+exports.getOutputBGMFilePath = getOutputBGMFilePath;
+const getOutputVideoFilePath = (outDirPath, fileName) => {
+    return path_1.default.resolve(outDirPath, fileName + ".mp4");
+};
+exports.getOutputVideoFilePath = getOutputVideoFilePath;
+const getOutputAudioFilePath = (outDirPath, fileName) => {
+    return path_1.default.resolve(outDirPath, fileName + ".mp3");
+};
+exports.getOutputAudioFilePath = getOutputAudioFilePath;
+const getScratchpadFilePath = (scratchpadDirName, fileName) => {
+    return path_1.default.resolve(scratchpadDirName, fileName + ".mp3");
+};
+exports.getScratchpadFilePath = getScratchpadFilePath;
+const getTemplateFilePath = (templateName) => {
+    return path_1.default.resolve(__dirname, "../../assets/templates/" + templateName + ".json");
+};
+exports.getTemplateFilePath = getTemplateFilePath;
+const mkdir = (dirPath) => {
+    if (!fs_1.default.existsSync(dirPath)) {
+        console.info("mkdir: " + dirPath);
+        fs_1.default.mkdirSync(dirPath, { recursive: true });
+    }
+};
+exports.mkdir = mkdir;
+exports.silentPath = path_1.default.resolve(__dirname, "../../assets/audio/silent300.mp3");
+exports.silentLastPath = path_1.default.resolve(__dirname, "../../assets/audio/silent800.mp3");
+exports.defaultBGMPath = path_1.default.resolve(__dirname, "../../assets/music/StarsBeyondEx.mp3");
+// for cli
+const getBaseDirPath = (basedir) => {
+    if (!basedir) {
+        return process.cwd();
+    }
+    if (path_1.default.isAbsolute(basedir)) {
+        return path_1.default.normalize(basedir);
+    }
+    return path_1.default.resolve(process.cwd(), basedir);
+};
+exports.getBaseDirPath = getBaseDirPath;
+const getFullPath = (baseDirPath, file) => {
+    if (path_1.default.isAbsolute(file)) {
+        return path_1.default.normalize(file);
+    }
+    if (baseDirPath) {
+        return path_1.default.resolve(baseDirPath, file);
+    }
+    return path_1.default.resolve(file);
+};
+exports.getFullPath = getFullPath;
+const readTemplatePrompt = (templateName) => {
+    const templatePath = (0, exports.getTemplateFilePath)(templateName);
+    const scriptData = fs_1.default.readFileSync(templatePath, "utf-8");
+    const template = schema_1.mulmoScriptTemplateSchema.parse(JSON.parse(scriptData));
+    const prompt = mulmo_script_template_1.MulmoScriptTemplateMethods.getSystemPrompt(template);
+    return prompt;
+};
+exports.readTemplatePrompt = readTemplatePrompt;
+const getAvailableTemplates = () => {
+    const templatesDir = path_1.default.resolve(__dirname, "../../assets/templates");
+    if (!fs_1.default.existsSync(templatesDir)) {
+        return [];
+    }
+    const files = fs_1.default.readdirSync(templatesDir);
+    return files.map((file) => {
+        const template = JSON.parse(fs_1.default.readFileSync(path_1.default.resolve(templatesDir, file), "utf-8"));
+        return {
+            ...schema_1.mulmoScriptTemplateSchema.parse(template),
+            filename: file.replace(/\.json$/, ""),
+        };
+    });
+};
+exports.getAvailableTemplates = getAvailableTemplates;
+const writingMessage = (filePath) => {
+    graphai_1.GraphAILogger.info(`writing: ${filePath}`);
+};
+exports.writingMessage = writingMessage;

package/lib/utils/filters.d.ts

@@ -0,0 +1,3 @@
+import "dotenv/config";
+import type { AgentFilterFunction } from "graphai";
+export declare const fileCacheAgentFilter: AgentFilterFunction;

package/lib/utils/filters.js

@@ -0,0 +1,32 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fileCacheAgentFilter = void 0;
+require("dotenv/config");
+const promises_1 = __importDefault(require("fs/promises"));
+const graphai_1 = require("graphai");
+const file_1 = require("./file");
+const fileCacheAgentFilter = async (context, next) => {
+    const { namedInputs } = context;
+    const { file, text } = namedInputs;
+    try {
+        await promises_1.default.access(file);
+        const elements = file.split("/");
+        graphai_1.GraphAILogger.info("cache hit: " + elements[elements.length - 1], text.slice(0, 10));
+        return true;
+    }
+    catch (__e) {
+        const output = (await next(context));
+        const buffer = output ? output["buffer"] : undefined;
+        if (buffer) {
+            (0, file_1.writingMessage)(file);
+            await promises_1.default.writeFile(file, buffer);
+            return true;
+        }
+        graphai_1.GraphAILogger.log("no cache, no buffer: " + file);
+        return false;
+    }
+};
+exports.fileCacheAgentFilter = fileCacheAgentFilter;

package/lib/utils/markdown.d.ts

@@ -0,0 +1 @@
+export declare const convertMarkdownToImage: (markdown: string, style: string, outputPath: string) => Promise<void>;

package/lib/utils/markdown.js

@@ -0,0 +1,27 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertMarkdownToImage = void 0;
+const marked_1 = require("marked");
+const puppeteer_1 = __importDefault(require("puppeteer"));
+const convertMarkdownToImage = async (markdown, style, outputPath) => {
+    // Step 0: Prepare the header
+    const header = `<head><style>${style}</style></head>`;
+    // Step 1: Convert Markdown to HTML
+    const body = await (0, marked_1.marked)(markdown);
+    const html = `<htlm>${header}<body>${body}</body></html>`;
+    // Step 2: Use Puppeteer to render HTML to an image
+    const browser = await puppeteer_1.default.launch();
+    const page = await browser.newPage();
+    // Set the page content to the HTML generated from the Markdown
+    await page.setContent(html);
+    // Adjust page settings if needed (like width, height, etc.)
+    await page.setViewport({ width: 1200, height: 800 });
+    // Step 3: Capture screenshot of the page (which contains the Markdown-rendered HTML)
+    await page.screenshot({ path: outputPath });
+    await browser.close();
+    console.log(`Image saved to ${outputPath}`);
+};
+exports.convertMarkdownToImage = convertMarkdownToImage;