mulmocast 2.0.0-alpha.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/actions/image_agents.js +1 -1
- package/lib/agents/movie_genai_agent.js +2 -4
- package/lib/agents/validate_schema_agent.d.ts +2 -3
- package/lib/types/schema.d.ts +89 -89
- package/lib/types/schema.js +11 -12
- package/lib/utils/agent_error_handler.d.ts +24 -0
- package/lib/utils/agent_error_handler.js +59 -0
- package/lib/utils/agent_info_factory.d.ts +12 -0
- package/lib/utils/agent_info_factory.js +23 -0
- package/lib/utils/context.d.ts +9 -9
- package/package.json +5 -5
- package/scripts/test/test_movie2.json +27 -0
- package/scripts/test/test_movie2.json~ +40 -0
- package/scripts/test/test_movie3.json +56 -0
- package/scripts/test/test_play_to_end.json~ +65 -0
- package/lib/agents/image_google_agent.d.ts +0 -5
- package/lib/agents/image_google_agent.js +0 -86
- package/lib/agents/movie_google_agent.d.ts +0 -9
- package/lib/agents/movie_google_agent.js +0 -122
- package/lib/mcp/aaa.d.ts +0 -2
- package/lib/mcp/aaa.js +0 -4
- package/lib/utils/aaa.d.ts +0 -1
- package/lib/utils/aaa.js +0 -16
- package/lib/utils/common.d.ts +0 -1
- package/lib/utils/common.js +0 -4
- package/lib/utils/greedy-json-lite.d.ts +0 -2
- package/lib/utils/greedy-json-lite.js +0 -59
- package/scripts/templates/html_sample.json~ +0 -0
- package/scripts/test/test_vision.json~ +0 -0
package/lib/types/schema.js
CHANGED
|
@@ -2,7 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders, provider2SoundEffectAgent, } from "../utils/provider2agent.js";
|
|
3
3
|
import { currentMulmoScriptVersion } from "../utils/const.js";
|
|
4
4
|
export const langSchema = z.string();
|
|
5
|
-
const URLStringSchema = z.
|
|
5
|
+
const URLStringSchema = z.url();
|
|
6
6
|
export const localizedTextSchema = z
|
|
7
7
|
.object({
|
|
8
8
|
text: z.string(),
|
|
@@ -278,6 +278,11 @@ export const mulmoLipSyncParamsSchema = z.object({
|
|
|
278
278
|
provider: z.string().optional(), // lip sync provider
|
|
279
279
|
model: z.string().optional(), // default: provider specific
|
|
280
280
|
});
|
|
281
|
+
const movieParamsSchema = z.object({
|
|
282
|
+
provider: text2MovieProviderSchema.optional(),
|
|
283
|
+
model: z.string().optional(),
|
|
284
|
+
fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
|
|
285
|
+
});
|
|
281
286
|
export const mulmoBeatSchema = z
|
|
282
287
|
.object({
|
|
283
288
|
speaker: speakerIdSchema.optional(),
|
|
@@ -289,11 +294,8 @@ export const mulmoBeatSchema = z
|
|
|
289
294
|
duration: z.number().optional().describe("Duration of the beat. Used only when the text is empty"),
|
|
290
295
|
imageParams: mulmoBeatImageParamsSchema.optional(), // beat specific parameters
|
|
291
296
|
audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
|
|
292
|
-
movieParams:
|
|
293
|
-
.
|
|
294
|
-
provider: text2MovieProviderSchema.optional(),
|
|
295
|
-
model: z.string().optional(),
|
|
296
|
-
fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
|
|
297
|
+
movieParams: movieParamsSchema
|
|
298
|
+
.extend({
|
|
297
299
|
speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."), // for movie.ts
|
|
298
300
|
})
|
|
299
301
|
.optional(),
|
|
@@ -344,12 +346,9 @@ export const mulmoTransitionSchema = z.object({
|
|
|
344
346
|
type: z.enum(["fade", "slideout_left"]),
|
|
345
347
|
duration: z.number().min(0).max(2).optional().default(0.3), // transition duration in seconds
|
|
346
348
|
});
|
|
347
|
-
export const mulmoMovieParamsSchema =
|
|
348
|
-
.
|
|
349
|
-
provider: text2MovieProviderSchema.optional(), // for agent
|
|
350
|
-
model: z.string().optional(), // default: provider specific. for agent
|
|
349
|
+
export const mulmoMovieParamsSchema = movieParamsSchema
|
|
350
|
+
.extend({
|
|
351
351
|
transition: mulmoTransitionSchema.optional(), // for movie.ts
|
|
352
|
-
fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
|
|
353
352
|
})
|
|
354
353
|
.strict();
|
|
355
354
|
export const mulmoPresentationStyleSchema = z.object({
|
|
@@ -493,4 +492,4 @@ export const mulmoStoryboardSchema = z
|
|
|
493
492
|
})
|
|
494
493
|
.describe("A storyboard for a presentation, a story, a video, etc.")
|
|
495
494
|
.strict();
|
|
496
|
-
export const urlsSchema = z.array(z.
|
|
495
|
+
export const urlsSchema = z.array(z.url({ message: "Invalid URL format" }));
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wraps agent execution with standardized error handling
|
|
3
|
+
*
|
|
4
|
+
* @param agentName - Name of the agent for error reporting
|
|
5
|
+
* @param action - Action being performed (e.g., "audio", "image", "movie")
|
|
6
|
+
* @param target - Target of the action (e.g., "audioFile", "imageFile")
|
|
7
|
+
* @param suppressError - If true, returns error object instead of throwing
|
|
8
|
+
* @param fn - Async function to execute
|
|
9
|
+
* @returns Result of fn or error object if suppressError is true
|
|
10
|
+
*/
|
|
11
|
+
export declare const wrapAgentError: <T>(agentName: string, action: string, target: string, suppressError: boolean | undefined, fn: () => Promise<T>) => Promise<T | {
|
|
12
|
+
error: unknown;
|
|
13
|
+
}>;
|
|
14
|
+
/**
|
|
15
|
+
* Validates that an API key is present, throwing an error if missing
|
|
16
|
+
*
|
|
17
|
+
* @param apiKey - API key to validate
|
|
18
|
+
* @param agentName - Name of the agent for error reporting
|
|
19
|
+
* @param action - Action being performed
|
|
20
|
+
* @param envVarName - Environment variable name for the API key
|
|
21
|
+
* @returns The validated API key
|
|
22
|
+
* @throws Error if API key is missing
|
|
23
|
+
*/
|
|
24
|
+
export declare const requireApiKey: (apiKey: string | undefined, agentName: string, action: string, envVarName: string) => string;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { GraphAILogger } from "graphai";
|
|
2
|
+
import { AuthenticationError, RateLimitError } from "openai";
|
|
3
|
+
import { agentIncorrectAPIKeyError, agentAPIRateLimitError, agentGenerationError, hasCause, } from "./error_cause.js";
|
|
4
|
+
/**
|
|
5
|
+
* Wraps agent execution with standardized error handling
|
|
6
|
+
*
|
|
7
|
+
* @param agentName - Name of the agent for error reporting
|
|
8
|
+
* @param action - Action being performed (e.g., "audio", "image", "movie")
|
|
9
|
+
* @param target - Target of the action (e.g., "audioFile", "imageFile")
|
|
10
|
+
* @param suppressError - If true, returns error object instead of throwing
|
|
11
|
+
* @param fn - Async function to execute
|
|
12
|
+
* @returns Result of fn or error object if suppressError is true
|
|
13
|
+
*/
|
|
14
|
+
export const wrapAgentError = async (agentName, action, target, suppressError, fn) => {
|
|
15
|
+
try {
|
|
16
|
+
return await fn();
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
if (suppressError) {
|
|
20
|
+
return { error };
|
|
21
|
+
}
|
|
22
|
+
GraphAILogger.info(error);
|
|
23
|
+
if (error instanceof AuthenticationError) {
|
|
24
|
+
throw new Error(`Failed: 401 Incorrect API key provided`, {
|
|
25
|
+
cause: agentIncorrectAPIKeyError(agentName, action, target),
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
if (error instanceof RateLimitError) {
|
|
29
|
+
throw new Error("You exceeded your current quota, please check your plan and billing details", {
|
|
30
|
+
cause: agentAPIRateLimitError(agentName, action, target),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
if (hasCause(error)) {
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
throw new Error(`Failed with ${agentName}`, {
|
|
37
|
+
cause: agentGenerationError(agentName, action, target),
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Validates that an API key is present, throwing an error if missing
|
|
43
|
+
*
|
|
44
|
+
* @param apiKey - API key to validate
|
|
45
|
+
* @param agentName - Name of the agent for error reporting
|
|
46
|
+
* @param action - Action being performed
|
|
47
|
+
* @param envVarName - Environment variable name for the API key
|
|
48
|
+
* @returns The validated API key
|
|
49
|
+
* @throws Error if API key is missing
|
|
50
|
+
*/
|
|
51
|
+
export const requireApiKey = (apiKey, agentName, action, envVarName) => {
|
|
52
|
+
if (!apiKey) {
|
|
53
|
+
const { apiKeyMissingError } = require("./error_cause.js");
|
|
54
|
+
throw new Error(`${envVarName} is required`, {
|
|
55
|
+
cause: apiKeyMissingError(agentName, action, envVarName),
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return apiKey;
|
|
59
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
|
+
/**
|
|
3
|
+
* Creates standardized AgentFunctionInfo metadata for agents
|
|
4
|
+
*
|
|
5
|
+
* @param name - Agent name (e.g., "ttsOpenaiAgent")
|
|
6
|
+
* @param agent - Agent function implementation
|
|
7
|
+
* @param category - Agent categories (e.g., ["tts"], ["image"])
|
|
8
|
+
* @param description - Human-readable description
|
|
9
|
+
* @param environmentVariables - Required environment variables (e.g., ["OPENAI_API_KEY"])
|
|
10
|
+
* @returns AgentFunctionInfo with standardized metadata
|
|
11
|
+
*/
|
|
12
|
+
export declare const createAgentInfo: <P = unknown, R = unknown, I = unknown, C = unknown>(name: string, agent: AgentFunction<P, R, I, C>, category: string[], description: string, environmentVariables?: string[]) => AgentFunctionInfo;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { PROJECT_METADATA } from "./const.js";
|
|
2
|
+
/**
|
|
3
|
+
* Creates standardized AgentFunctionInfo metadata for agents
|
|
4
|
+
*
|
|
5
|
+
* @param name - Agent name (e.g., "ttsOpenaiAgent")
|
|
6
|
+
* @param agent - Agent function implementation
|
|
7
|
+
* @param category - Agent categories (e.g., ["tts"], ["image"])
|
|
8
|
+
* @param description - Human-readable description
|
|
9
|
+
* @param environmentVariables - Required environment variables (e.g., ["OPENAI_API_KEY"])
|
|
10
|
+
* @returns AgentFunctionInfo with standardized metadata
|
|
11
|
+
*/
|
|
12
|
+
export const createAgentInfo = (name, agent, category, description, environmentVariables = []) => ({
|
|
13
|
+
name,
|
|
14
|
+
agent,
|
|
15
|
+
mock: agent,
|
|
16
|
+
samples: [],
|
|
17
|
+
description,
|
|
18
|
+
category,
|
|
19
|
+
author: PROJECT_METADATA.author,
|
|
20
|
+
repository: PROJECT_METADATA.repository,
|
|
21
|
+
license: PROJECT_METADATA.license,
|
|
22
|
+
environmentVariables,
|
|
23
|
+
});
|
package/lib/utils/context.d.ts
CHANGED
|
@@ -255,13 +255,13 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
255
255
|
movieParams?: {
|
|
256
256
|
provider?: string | undefined;
|
|
257
257
|
model?: string | undefined;
|
|
258
|
+
fillOption?: {
|
|
259
|
+
style: "aspectFit" | "aspectFill";
|
|
260
|
+
} | undefined;
|
|
258
261
|
transition?: {
|
|
259
262
|
type: "fade" | "slideout_left";
|
|
260
263
|
duration: number;
|
|
261
264
|
} | undefined;
|
|
262
|
-
fillOption?: {
|
|
263
|
-
style: "aspectFit" | "aspectFill";
|
|
264
|
-
} | undefined;
|
|
265
265
|
} | undefined;
|
|
266
266
|
lipSyncParams?: {
|
|
267
267
|
provider?: string | undefined;
|
|
@@ -570,13 +570,13 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
570
570
|
movieParams?: {
|
|
571
571
|
provider?: string | undefined;
|
|
572
572
|
model?: string | undefined;
|
|
573
|
+
fillOption?: {
|
|
574
|
+
style: "aspectFit" | "aspectFill";
|
|
575
|
+
} | undefined;
|
|
573
576
|
transition?: {
|
|
574
577
|
type: "fade" | "slideout_left";
|
|
575
578
|
duration: number;
|
|
576
579
|
} | undefined;
|
|
577
|
-
fillOption?: {
|
|
578
|
-
style: "aspectFit" | "aspectFill";
|
|
579
|
-
} | undefined;
|
|
580
580
|
} | undefined;
|
|
581
581
|
lipSyncParams?: {
|
|
582
582
|
provider?: string | undefined;
|
|
@@ -719,13 +719,13 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
719
719
|
movieParams?: {
|
|
720
720
|
provider?: string | undefined;
|
|
721
721
|
model?: string | undefined;
|
|
722
|
+
fillOption?: {
|
|
723
|
+
style: "aspectFit" | "aspectFill";
|
|
724
|
+
} | undefined;
|
|
722
725
|
transition?: {
|
|
723
726
|
type: "fade" | "slideout_left";
|
|
724
727
|
duration: number;
|
|
725
728
|
} | undefined;
|
|
726
|
-
fillOption?: {
|
|
727
|
-
style: "aspectFit" | "aspectFill";
|
|
728
|
-
} | undefined;
|
|
729
729
|
} | undefined;
|
|
730
730
|
lipSyncParams?: {
|
|
731
731
|
provider?: string | undefined;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.1",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.node.js",
|
|
@@ -84,13 +84,13 @@
|
|
|
84
84
|
"@graphai/stream_agent_filter": "^2.0.2",
|
|
85
85
|
"@graphai/vanilla": "^2.0.12",
|
|
86
86
|
"@graphai/vanilla_node_agents": "^2.0.4",
|
|
87
|
-
"@inquirer/input": "^4.
|
|
88
|
-
"@inquirer/select": "^4.4.
|
|
89
|
-
"@modelcontextprotocol/sdk": "^1.21.
|
|
87
|
+
"@inquirer/input": "^4.3.0",
|
|
88
|
+
"@inquirer/select": "^4.4.1",
|
|
89
|
+
"@modelcontextprotocol/sdk": "^1.21.1",
|
|
90
90
|
"@mozilla/readability": "^0.6.0",
|
|
91
91
|
"@tavily/core": "^0.5.11",
|
|
92
92
|
"archiver": "^7.0.1",
|
|
93
|
-
"clipboardy": "^
|
|
93
|
+
"clipboardy": "^5.0.0",
|
|
94
94
|
"dotenv": "^17.2.3",
|
|
95
95
|
"fluent-ffmpeg": "^2.1.3",
|
|
96
96
|
"graphai": "^2.0.16",
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"movieParams": {
|
|
7
|
+
"provider": "google"
|
|
8
|
+
},
|
|
9
|
+
"canvasSize": {
|
|
10
|
+
"width": 1280,
|
|
11
|
+
"height": 720
|
|
12
|
+
},
|
|
13
|
+
"beats": [
|
|
14
|
+
{
|
|
15
|
+
"text": "Hello, I'm macoro",
|
|
16
|
+
"image": {
|
|
17
|
+
"type": "image",
|
|
18
|
+
"source": {
|
|
19
|
+
"kind": "url",
|
|
20
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/macoro.png"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"moviePrompt": "Jump and bounce",
|
|
24
|
+
"duration": 5
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"movieParams": {
|
|
7
|
+
"provider": "google"
|
|
8
|
+
},
|
|
9
|
+
"canvasSize": {
|
|
10
|
+
"width": 720,
|
|
11
|
+
"height": 1280
|
|
12
|
+
},
|
|
13
|
+
"imageParams": {
|
|
14
|
+
"provider": "openai",
|
|
15
|
+
"model": "dall-e-3",
|
|
16
|
+
"style": "Photo realistic, cinematic style.",
|
|
17
|
+
"images": {
|
|
18
|
+
"optimus": {
|
|
19
|
+
"type": "image",
|
|
20
|
+
"source": {
|
|
21
|
+
"kind": "url",
|
|
22
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/optimus.png"
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"beats": [
|
|
28
|
+
{
|
|
29
|
+
"text": "Image with both image and movie prompt",
|
|
30
|
+
"imagePrompt": "巨大なロケット「Starship」が発射台から打ち上がろうとしている様子。空には星と火星が浮かんでいる。",
|
|
31
|
+
"moviePrompt": "Startship lifts off.",
|
|
32
|
+
"duration": 5
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"text": "Image with only movie prompt",
|
|
36
|
+
"moviePrompt": "Startship lifts off.",
|
|
37
|
+
"duration": 5
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"movieParams": {
|
|
7
|
+
"provider": "google"
|
|
8
|
+
},
|
|
9
|
+
"lipSyncParams": {
|
|
10
|
+
"provider": "replicate",
|
|
11
|
+
"model": "bytedance/latentsync"
|
|
12
|
+
},
|
|
13
|
+
"canvasSize": {
|
|
14
|
+
"width": 1280,
|
|
15
|
+
"height": 720
|
|
16
|
+
},
|
|
17
|
+
"beats": [
|
|
18
|
+
{
|
|
19
|
+
"text": "Hello, I'm macoro",
|
|
20
|
+
"image": {
|
|
21
|
+
"type": "image",
|
|
22
|
+
"source": {
|
|
23
|
+
"kind": "url",
|
|
24
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/macoro.png"
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"moviePrompt": "Jump and bounce",
|
|
28
|
+
"duration": 5
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"text": "こんにちは、マコロです",
|
|
32
|
+
"image": {
|
|
33
|
+
"type": "image",
|
|
34
|
+
"source": {
|
|
35
|
+
"kind": "url",
|
|
36
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/macoro.png"
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"enableLipSync": true,
|
|
40
|
+
"duration": 5
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"text": "こんにちは、マコロです",
|
|
44
|
+
"image": {
|
|
45
|
+
"type": "image",
|
|
46
|
+
"source": {
|
|
47
|
+
"kind": "url",
|
|
48
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/macoro.png"
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
"moviePrompt": "Jump and bounce",
|
|
52
|
+
"enableLipSync": true,
|
|
53
|
+
"duration": 5
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1",
|
|
4
|
+
"credit": "closing"
|
|
5
|
+
},
|
|
6
|
+
"canvasSize": {
|
|
7
|
+
"width": 1536,
|
|
8
|
+
"height": 1024
|
|
9
|
+
},
|
|
10
|
+
"speechParams": {
|
|
11
|
+
"speakers": {
|
|
12
|
+
"Presenter": {
|
|
13
|
+
"displayName": {
|
|
14
|
+
"en": "Presenter"
|
|
15
|
+
},
|
|
16
|
+
"voiceId": "shimmer"
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"imageParams": {
|
|
21
|
+
"provider": "openai",
|
|
22
|
+
"style": "<style>Vibrant 3D animation style inspired by K-pop aesthetics, with glossy, stylized characters. The overall visual style combines elements of modern animation, game cinematics, and fashion-forward character design, with sleek outlines, glowing effects, and a polished, cinematic finish.</style>",
|
|
23
|
+
"images": {
|
|
24
|
+
"min": {
|
|
25
|
+
"type": "image",
|
|
26
|
+
"source": {
|
|
27
|
+
"kind": "url",
|
|
28
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/min_anime.png"
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"movieParams": {
|
|
34
|
+
"provider": "replicate"
|
|
35
|
+
},
|
|
36
|
+
"soundEffectParams": {
|
|
37
|
+
"provider": "replicate"
|
|
38
|
+
},
|
|
39
|
+
"captionParams": {
|
|
40
|
+
"lang": "en",
|
|
41
|
+
"styles": ["font-size: 64px", "width: 90%", "padding-left: 5%", "padding-right: 5%"]
|
|
42
|
+
},
|
|
43
|
+
"audioParams": {
|
|
44
|
+
"padding": 0,
|
|
45
|
+
"introPadding": 0,
|
|
46
|
+
"closingPadding": 0,
|
|
47
|
+
"outroPadding": 0,
|
|
48
|
+
"bgm": {
|
|
49
|
+
"kind": "url",
|
|
50
|
+
"url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/music/finetuning_with_you.mp3"
|
|
51
|
+
},
|
|
52
|
+
"bgmVolume": 1,
|
|
53
|
+
"audioVolume": 0,
|
|
54
|
+
"suppressSpeech": true
|
|
55
|
+
},
|
|
56
|
+
"title": "Music Video",
|
|
57
|
+
"lang": "en",
|
|
58
|
+
"beats": [
|
|
59
|
+
{
|
|
60
|
+
"text": "Finetuning with you",
|
|
61
|
+
"moviePrompt": "The singer preparing to sing a song.",
|
|
62
|
+
"playGenMovieToEnd": true
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
|
-
import type { AgentBufferResult, ImageAgentInputs, ImageAgentParams, GoogleImageAgentConfig } from "../types/agent.js";
|
|
3
|
-
export declare const imageGoogleAgent: AgentFunction<ImageAgentParams, AgentBufferResult, ImageAgentInputs, GoogleImageAgentConfig>;
|
|
4
|
-
declare const imageGoogleAgentInfo: AgentFunctionInfo;
|
|
5
|
-
export default imageGoogleAgentInfo;
|
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
import { GraphAILogger } from "graphai";
|
|
2
|
-
import { getAspectRatio } from "./movie_google_agent.js";
|
|
3
|
-
import { provider2ImageAgent } from "../utils/provider2agent.js";
|
|
4
|
-
async function generateImage(projectId, model, token, prompt, aspectRatio) {
|
|
5
|
-
const GOOGLE_IMAGEN_ENDPOINT = `https://us-central1-aiplatform.googleapis.com/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${model}:predict`;
|
|
6
|
-
try {
|
|
7
|
-
// Prepare the payload for the API request
|
|
8
|
-
const payload = {
|
|
9
|
-
instances: [
|
|
10
|
-
{
|
|
11
|
-
prompt,
|
|
12
|
-
},
|
|
13
|
-
],
|
|
14
|
-
parameters: {
|
|
15
|
-
sampleCount: 1,
|
|
16
|
-
aspectRatio,
|
|
17
|
-
safetySetting: "block_only_high",
|
|
18
|
-
},
|
|
19
|
-
};
|
|
20
|
-
// Make the API call using fetch
|
|
21
|
-
const response = await fetch(GOOGLE_IMAGEN_ENDPOINT, {
|
|
22
|
-
method: "POST",
|
|
23
|
-
headers: {
|
|
24
|
-
Authorization: `Bearer ${token}`,
|
|
25
|
-
"Content-Type": "application/json",
|
|
26
|
-
},
|
|
27
|
-
body: JSON.stringify(payload),
|
|
28
|
-
});
|
|
29
|
-
if (!response.ok) {
|
|
30
|
-
throw new Error(`Error: ${response.status} - ${response.statusText}`);
|
|
31
|
-
}
|
|
32
|
-
const responseData = await response.json();
|
|
33
|
-
// Parse and return the generated image URL or data
|
|
34
|
-
const predictions = responseData.predictions;
|
|
35
|
-
if (predictions && predictions.length > 0) {
|
|
36
|
-
const base64Image = predictions[0].bytesBase64Encoded;
|
|
37
|
-
if (base64Image) {
|
|
38
|
-
return Buffer.from(base64Image, "base64"); // Decode the base64 image to a buffer
|
|
39
|
-
}
|
|
40
|
-
else {
|
|
41
|
-
throw new Error("No base64-encoded image data returned from the API.");
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
else {
|
|
45
|
-
// console.log(response);
|
|
46
|
-
GraphAILogger.info("No predictions returned from the API.", responseData, prompt);
|
|
47
|
-
return undefined;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
catch (error) {
|
|
51
|
-
GraphAILogger.info("Error generating image:", error);
|
|
52
|
-
throw error;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
export const imageGoogleAgent = async ({ namedInputs, params, config, }) => {
|
|
56
|
-
const { prompt } = namedInputs;
|
|
57
|
-
const aspectRatio = getAspectRatio(params.canvasSize);
|
|
58
|
-
const model = params.model ?? provider2ImageAgent["google"].defaultModel;
|
|
59
|
-
const projectId = config?.projectId;
|
|
60
|
-
const token = config?.token;
|
|
61
|
-
try {
|
|
62
|
-
const buffer = await generateImage(projectId, model, token, prompt, aspectRatio);
|
|
63
|
-
if (buffer) {
|
|
64
|
-
return { buffer };
|
|
65
|
-
}
|
|
66
|
-
throw new Error("ERROR: geneateImage returned undefined");
|
|
67
|
-
}
|
|
68
|
-
catch (error) {
|
|
69
|
-
GraphAILogger.info("Failed to generate image:", error);
|
|
70
|
-
throw error;
|
|
71
|
-
}
|
|
72
|
-
};
|
|
73
|
-
const imageGoogleAgentInfo = {
|
|
74
|
-
name: "imageGoogleAgent",
|
|
75
|
-
agent: imageGoogleAgent,
|
|
76
|
-
mock: imageGoogleAgent,
|
|
77
|
-
samples: [],
|
|
78
|
-
description: "Google Image agent",
|
|
79
|
-
category: ["image"],
|
|
80
|
-
author: "Receptron Team",
|
|
81
|
-
repository: "https://github.com/receptron/mulmocast-cli/",
|
|
82
|
-
// source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
|
|
83
|
-
license: "MIT",
|
|
84
|
-
environmentVariables: [],
|
|
85
|
-
};
|
|
86
|
-
export default imageGoogleAgentInfo;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
|
-
import type { AgentBufferResult, GoogleImageAgentConfig, GoogleMovieAgentParams, MovieAgentInputs } from "../types/agent.js";
|
|
3
|
-
export declare const getAspectRatio: (canvasSize: {
|
|
4
|
-
width: number;
|
|
5
|
-
height: number;
|
|
6
|
-
}) => string;
|
|
7
|
-
export declare const movieGoogleAgent: AgentFunction<GoogleMovieAgentParams, AgentBufferResult, MovieAgentInputs, GoogleImageAgentConfig>;
|
|
8
|
-
declare const movieGoogleAgentInfo: AgentFunctionInfo;
|
|
9
|
-
export default movieGoogleAgentInfo;
|
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from "fs";
|
|
2
|
-
import { GraphAILogger, sleep } from "graphai";
|
|
3
|
-
async function generateMovie(projectId, model, token, prompt, imagePath, aspectRatio, duration) {
|
|
4
|
-
const GOOGLE_IMAGEN_ENDPOINT = `https://us-central1-aiplatform.googleapis.com/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${model}`;
|
|
5
|
-
const payload = {
|
|
6
|
-
instances: [
|
|
7
|
-
{
|
|
8
|
-
prompt,
|
|
9
|
-
image: undefined,
|
|
10
|
-
},
|
|
11
|
-
],
|
|
12
|
-
parameters: {
|
|
13
|
-
sampleCount: 1,
|
|
14
|
-
aspectRatio,
|
|
15
|
-
safetySetting: "block_only_high",
|
|
16
|
-
personGeneration: "allow_all",
|
|
17
|
-
durationSeconds: duration,
|
|
18
|
-
},
|
|
19
|
-
};
|
|
20
|
-
if (imagePath) {
|
|
21
|
-
const buffer = readFileSync(imagePath);
|
|
22
|
-
const bytesBase64Encoded = buffer.toString("base64");
|
|
23
|
-
payload.instances[0].image = {
|
|
24
|
-
bytesBase64Encoded,
|
|
25
|
-
mimeType: "image/png",
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
|
-
// Make the API call using fetch
|
|
29
|
-
const response = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:predictLongRunning`, {
|
|
30
|
-
method: "POST",
|
|
31
|
-
headers: {
|
|
32
|
-
Authorization: `Bearer ${token}`,
|
|
33
|
-
"Content-Type": "application/json",
|
|
34
|
-
},
|
|
35
|
-
body: JSON.stringify(payload),
|
|
36
|
-
});
|
|
37
|
-
if (!response.ok) {
|
|
38
|
-
GraphAILogger.info("create project on google cloud console and setup the project. More details see readme.");
|
|
39
|
-
throw new Error(`Error: ${response.status} - ${response.statusText}`);
|
|
40
|
-
}
|
|
41
|
-
const initialResponse = await response.json();
|
|
42
|
-
const fetchBody = {
|
|
43
|
-
operationName: initialResponse.name,
|
|
44
|
-
};
|
|
45
|
-
const completeResponse = await (async () => {
|
|
46
|
-
while (true) {
|
|
47
|
-
GraphAILogger.info("...waiting for movie generation...");
|
|
48
|
-
await sleep(3000);
|
|
49
|
-
const operationResponse = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:fetchPredictOperation`, {
|
|
50
|
-
method: "POST",
|
|
51
|
-
headers: {
|
|
52
|
-
Authorization: `Bearer ${token}`,
|
|
53
|
-
"Content-Type": "application/json",
|
|
54
|
-
},
|
|
55
|
-
body: JSON.stringify(fetchBody),
|
|
56
|
-
});
|
|
57
|
-
if (!operationResponse.ok) {
|
|
58
|
-
throw new Error(`Error: ${operationResponse.status} - ${operationResponse.statusText}`);
|
|
59
|
-
}
|
|
60
|
-
const responseData = await operationResponse.json();
|
|
61
|
-
if (responseData.done) {
|
|
62
|
-
if (responseData.error) {
|
|
63
|
-
GraphAILogger.info("Prompt: ", prompt);
|
|
64
|
-
throw new Error(`Error: ${responseData.error.message}`);
|
|
65
|
-
}
|
|
66
|
-
if (!responseData.response.videos) {
|
|
67
|
-
throw new Error(`No video: ${JSON.stringify(responseData, null, 2)}`);
|
|
68
|
-
}
|
|
69
|
-
return responseData.response;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
})();
|
|
73
|
-
const encodedMovie = completeResponse.videos[0].bytesBase64Encoded;
|
|
74
|
-
if (encodedMovie) {
|
|
75
|
-
return Buffer.from(encodedMovie, "base64");
|
|
76
|
-
}
|
|
77
|
-
return undefined;
|
|
78
|
-
}
|
|
79
|
-
export const getAspectRatio = (canvasSize) => {
|
|
80
|
-
if (canvasSize.width > canvasSize.height) {
|
|
81
|
-
return "16:9";
|
|
82
|
-
}
|
|
83
|
-
else if (canvasSize.width < canvasSize.height) {
|
|
84
|
-
return "9:16";
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
return "1:1";
|
|
88
|
-
}
|
|
89
|
-
};
|
|
90
|
-
export const movieGoogleAgent = async ({ namedInputs, params, config, }) => {
|
|
91
|
-
const { prompt, imagePath } = namedInputs;
|
|
92
|
-
const aspectRatio = getAspectRatio(params.canvasSize);
|
|
93
|
-
const model = params.model ?? "veo-2.0-generate-001"; // "veo-3.0-generate-preview";
|
|
94
|
-
const duration = params.duration ?? 8;
|
|
95
|
-
const projectId = config?.projectId;
|
|
96
|
-
const token = config?.token;
|
|
97
|
-
try {
|
|
98
|
-
const buffer = await generateMovie(projectId, model, token, prompt, imagePath, aspectRatio, duration);
|
|
99
|
-
if (buffer) {
|
|
100
|
-
return { buffer };
|
|
101
|
-
}
|
|
102
|
-
throw new Error("ERROR: geneateImage returned undefined");
|
|
103
|
-
}
|
|
104
|
-
catch (error) {
|
|
105
|
-
GraphAILogger.info("Failed to generate movie:", error.message);
|
|
106
|
-
throw error;
|
|
107
|
-
}
|
|
108
|
-
};
|
|
109
|
-
const movieGoogleAgentInfo = {
|
|
110
|
-
name: "movieGoogleAgent",
|
|
111
|
-
agent: movieGoogleAgent,
|
|
112
|
-
mock: movieGoogleAgent,
|
|
113
|
-
samples: [],
|
|
114
|
-
description: "Google Movie agent",
|
|
115
|
-
category: ["movie"],
|
|
116
|
-
author: "Receptron Team",
|
|
117
|
-
repository: "https://github.com/receptron/mulmocast-cli/",
|
|
118
|
-
// source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
|
|
119
|
-
license: "MIT",
|
|
120
|
-
environmentVariables: [],
|
|
121
|
-
};
|
|
122
|
-
export default movieGoogleAgentInfo;
|
package/lib/mcp/aaa.d.ts
DELETED