mulmocast 0.0.8 → 0.0.10

package/lib/agents/image_openai_agent.js

@@ -1,19 +1,31 @@
-import OpenAI from "openai";
+import fs from "fs";
+import OpenAI, { toFile } from "openai";
 // https://platform.openai.com/docs/guides/image-generation
 export const imageOpenaiAgent = async ({ namedInputs, params }) => {
     const { prompt } = namedInputs;
-    const { apiKey, model, size, moderation } = params;
+    const { apiKey, model, size, moderation, images } = params;
     const openai = new OpenAI({ apiKey });
     const imageOptions = {
         model: model ?? "dall-e-3",
         prompt,
         n: 1,
-        size: size || model === "gpt-image-1" ? "1536x1024" : "1792x1024",
+        size: size ?? (model === "gpt-image-1" ? "1536x1024" : "1792x1024"),
     };
     if (model === "gpt-image-1") {
         imageOptions.moderation = moderation || "auto";
     }
-    const response = await openai.images.generate(imageOptions);
+    const response = await (async () => {
+        const targetSize = imageOptions.size;
+        if ((images ?? []).length > 0 && (targetSize === "1536x1024" || targetSize === "1024x1536" || targetSize === "1024x1024")) {
+            const imagelist = await Promise.all((images ?? []).map(async (file) => await toFile(fs.createReadStream(file), null, {
+                type: "image/png", // TODO: Support JPEG as well
+            })));
+            return await openai.images.edit({ ...imageOptions, size: targetSize, image: imagelist });
+        }
+        else {
+            return await openai.images.generate(imageOptions);
+        }
+    })();
     if (!response.data) {
         throw new Error(`response.data is undefined: ${response}`);
     }

package/lib/agents/movie_google_agent.d.ts

@@ -0,0 +1,17 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+export type MovieGoogleConfig = {
+    projectId?: string;
+    token?: string;
+};
+export declare const movieGoogleAgent: AgentFunction<{
+    model: string;
+    aspectRatio: string;
+    duration?: number;
+}, {
+    buffer: Buffer;
+}, {
+    prompt: string;
+    imagePath: string;
+}, MovieGoogleConfig>;
+declare const movieGoogleAgentInfo: AgentFunctionInfo;
+export default movieGoogleAgentInfo;

package/lib/agents/movie_google_agent.js

@@ -0,0 +1,114 @@
+import { readFileSync } from "fs";
+import { GraphAILogger, sleep } from "graphai";
+async function generateMovie(projectId, model, token, prompt, imagePath, aspectRatio, duration) {
+    const GOOGLE_IMAGEN_ENDPOINT = `https://us-central1-aiplatform.googleapis.com/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${model}`;
+    // Prepare the payload for the API request
+    const buffer = readFileSync(imagePath);
+    const bytesBase64Encoded = buffer.toString("base64");
+    const payload = {
+        instances: [
+            {
+                prompt: prompt,
+                image: {
+                    bytesBase64Encoded,
+                    mimeType: "image/png",
+                },
+            },
+        ],
+        parameters: {
+            sampleCount: 1,
+            aspectRatio: aspectRatio,
+            //safetySetting: "block_only_high",
+            durationSeconds: duration,
+        },
+    };
+    // Make the API call using fetch
+    const response = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:predictLongRunning`, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify(payload),
+    });
+    if (!response.ok) {
+        throw new Error(`Error: ${response.status} - ${response.statusText}`);
+    }
+    const initialResponse = await response.json();
+    const fetchBody = {
+        operationName: initialResponse.name,
+    };
+    const completeResponse = await (async () => {
+        while (true) {
+            GraphAILogger.info("...waiting for movie generation...");
+            await sleep(3000);
+            const response = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:fetchPredictOperation`, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(fetchBody),
+            });
+            if (!response.ok) {
+                throw new Error(`Error: ${response.status} - ${response.statusText}`);
+            }
+            const responseData = await response.json();
+            if (responseData.done) {
+                if (responseData.error) {
+                    GraphAILogger.info("Prompt: ", prompt);
+                    throw new Error(`Error: ${responseData.error.message}`);
+                }
+                if (!responseData.response.videos) {
+                    throw new Error(`No video: ${JSON.stringify(responseData, null, 2)}`);
+                }
+                return responseData.response;
+            }
+        }
+    })();
+    const encodedMovie = completeResponse.videos[0].bytesBase64Encoded;
+    if (encodedMovie) {
+        return Buffer.from(encodedMovie, "base64");
+    }
+    return undefined;
+}
+export const movieGoogleAgent = async ({ namedInputs, params, config }) => {
+    const { prompt, imagePath } = namedInputs;
+    /*
+    if (prompt) {
+      const buffer = Buffer.from(prompt);
+      return { buffer };
+    }
+    */
+    const aspectRatio = params.aspectRatio ?? "16:9";
+    const model = params.model ?? "veo-2.0-generate-001"; // "veo-3.0-generate-preview";
+    const duration = params.duration ?? 8;
+    //const projectId = process.env.GOOGLE_PROJECT_ID; // Your Google Cloud Project ID
+    const projectId = config?.projectId;
+    const token = config?.token;
+    try {
+        const buffer = await generateMovie(projectId, model, token, prompt, imagePath, aspectRatio, duration);
+        if (buffer) {
+            return { buffer };
+        }
+        throw new Error("ERROR: geneateImage returned undefined");
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate movie:", error);
+        throw error;
+    }
+};
+const movieGoogleAgentInfo = {
+    name: "movieGoogleAgent",
+    agent: movieGoogleAgent,
+    mock: movieGoogleAgent,
+    samples: [],
+    description: "Google Movie agent",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    // source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
+    license: "MIT",
+    environmentVariables: [],
+};
+export default movieGoogleAgentInfo;

package/lib/cli/bin.js

@@ -2,6 +2,9 @@
 import "dotenv/config";
 import yargs from "yargs/yargs";
 import { hideBin } from "yargs/helpers";
+import { readFileSync } from "fs";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
 import * as translateCmd from "./commands/translate/index.js";
 import * as audioCmd from "./commands/audio/index.js";
 import * as imagesCmd from "./commands/image/index.js";
@@ -9,9 +12,13 @@ import * as movieCmd from "./commands/movie/index.js";
 import * as pdfCmd from "./commands/pdf/index.js";
 import * as toolCmd from "./commands/tool/index.js";
 import { GraphAILogger } from "graphai";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageJson = JSON.parse(readFileSync(join(__dirname, "../../package.json"), "utf8"));
 export const main = async () => {
     const cli = yargs(hideBin(process.argv))
         .scriptName("mulmo")
+        .version(packageJson.version)
         .usage("$0 <command> [options]")
         .option("v", {
         alias: "verbose",
@@ -29,6 +36,18 @@ export const main = async () => {
         .demandCommand()
         .strict()
         .help()
+        .showHelpOnFail(false)
+        .fail((msg, err, y) => {
+        // if yargs detect error, show help and exit
+        if (msg) {
+            y.showHelp();
+            GraphAILogger.info("\\n" + msg);
+            process.exit(1);
+        }
+        if (err) {
+            throw err;
+        }
+    })
         .alias("help", "h");
     await cli.parseAsync();
 };

package/lib/cli/helpers.js

@@ -2,7 +2,7 @@ import { GraphAILogger } from "graphai";
 import fs from "fs";
 import path from "path";
 import clipboardy from "clipboardy";
-import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath, resolveDirPath } from "../utils/file.js";
+import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath, resolveDirPath, mkdir } from "../utils/file.js";
 import { isHttp } from "../utils/utils.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
@@ -33,6 +33,7 @@ export const getFileObject = (args) => {
             const fileName = `script_${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
             const clipboardText = clipboardy.readSync();
             const fileOrUrl = resolveDirPath(outDirPath, `${fileName}.json`);
+            mkdir(outDirPath);
             fs.writeFileSync(fileOrUrl, clipboardText, "utf8");
             return { fileOrUrl, fileName };
         }

package/lib/methods/mulmo_studio.d.ts

@@ -1,6 +1,6 @@
 import { MulmoStudio } from "../types/index.js";
 type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-type BeatSessionType = "audio" | "image" | "multiLingual" | "caption";
+type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
 export declare const MulmoStudioMethods: {
     setSessionState(studio: MulmoStudio, sessionType: SessionType, value: boolean): void;
     setBeatSessionState(studio: MulmoStudio, sessionType: BeatSessionType, index: number, value: boolean): void;

package/lib/tools/create_mulmo_script_from_url.js

@@ -14,7 +14,7 @@ import { mulmoScriptSchema, urlsSchema } from "../types/schema.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
 import { graphDataScriptFromUrlPrompt } from "../utils/prompt.js";
 import { llmPair } from "../utils/utils.js";
-const { default: __, ...vanillaAgents } = agents;
+const vanillaAgents = agents.default ?? agents;
 const graphData = {
     version: 0.5,
     // Execute sequentially because the free version of browserless API doesn't support concurrent execution.

package/lib/tools/create_mulmo_script_interactively.js

@@ -16,7 +16,7 @@ import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { llmPair } from "../utils/utils.js";
 import { interactiveClarificationPrompt, prefixPrompt } from "../utils/prompt.js";
 // import { cliLoadingPlugin } from "../utils/plugins.js";
-const { default: __, ...vanillaAgents } = agents;
+const vanillaAgents = agents.default ?? agents;
 const agentHeader = "\x1b[34m● \x1b[0m\x1b[1mAgent\x1b[0m:\x1b[0m";
 const graphDataForScraping = {
     version: 0.5,

package/lib/tools/story_to_script.js

@@ -12,7 +12,7 @@ import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { llmPair } from "../utils/utils.js";
 import { storyToScriptGenerateMode } from "../utils/const.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
-const { default: __, ...vanillaAgents } = agents;
+const vanillaAgents = agents.default ?? agents;
 const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent, llmModel, maxTokens, }) => {
     return {
         loop: {