mulmocast 1.2.27 → 1.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/templates/documentary.json +19 -0
- package/lib/actions/image_agents.d.ts +16 -31
- package/lib/actions/image_agents.js +6 -5
- package/lib/actions/image_references.js +2 -2
- package/lib/actions/images.d.ts +17 -30
- package/lib/actions/images.js +4 -1
- package/lib/actions/index.d.ts +1 -0
- package/lib/actions/index.js +1 -0
- package/lib/actions/markdown.d.ts +3 -0
- package/lib/actions/markdown.js +61 -0
- package/lib/agents/image_replicate_agent.d.ts +7 -0
- package/lib/agents/image_replicate_agent.js +59 -0
- package/lib/agents/index.d.ts +2 -1
- package/lib/agents/index.js +2 -1
- package/lib/agents/movie_replicate_agent.js +9 -7
- package/lib/agents/puppeteer_crawler_agent.d.ts +0 -9
- package/lib/agents/puppeteer_crawler_agent.js +1 -1
- package/lib/cli/bin.js +2 -0
- package/lib/cli/commands/markdown/builder.d.ts +16 -0
- package/lib/cli/commands/markdown/builder.js +5 -0
- package/lib/cli/commands/markdown/handler.d.ts +4 -0
- package/lib/cli/commands/markdown/handler.js +11 -0
- package/lib/cli/commands/markdown/index.d.ts +4 -0
- package/lib/cli/commands/markdown/index.js +4 -0
- package/lib/data/promptTemplates.js +42 -0
- package/lib/data/scriptTemplates.js +160 -160
- package/lib/data/templateDataSet.d.ts +1 -0
- package/lib/data/templateDataSet.js +5 -1
- package/lib/mcp/aaa.js +1 -1
- package/lib/methods/mulmo_beat.d.ts +6 -1
- package/lib/types/agent.d.ts +12 -0
- package/lib/types/schema.d.ts +13 -0
- package/lib/types/schema.js +2 -0
- package/lib/types/type.d.ts +1 -1
- package/lib/utils/context.d.ts +3 -0
- package/lib/utils/context.js +1 -0
- package/lib/utils/image_plugins/image.d.ts +1 -1
- package/lib/utils/image_plugins/image.js +2 -2
- package/lib/utils/image_plugins/index.d.ts +7 -11
- package/lib/utils/image_plugins/index.js +1 -12
- package/lib/utils/image_plugins/markdown.d.ts +1 -0
- package/lib/utils/image_plugins/markdown.js +8 -1
- package/lib/utils/image_plugins/mermaid.d.ts +1 -0
- package/lib/utils/image_plugins/mermaid.js +9 -0
- package/lib/utils/image_plugins/movie.d.ts +1 -1
- package/lib/utils/image_plugins/movie.js +2 -2
- package/lib/utils/image_plugins/source.d.ts +2 -1
- package/lib/utils/image_plugins/source.js +34 -1
- package/lib/utils/image_plugins/text_slide.d.ts +1 -0
- package/lib/utils/image_plugins/text_slide.js +12 -1
- package/lib/utils/image_plugins/vision.js +1 -1
- package/lib/utils/provider2agent.d.ts +5 -0
- package/lib/utils/provider2agent.js +5 -0
- package/lib/utils/utils.js +3 -0
- package/package.json +5 -4
- package/scripts/templates/vision.json +160 -160
- package/scripts/test/test_image_refs.json +10 -0
- package/scripts/test/test_markdown.json +60 -0
- package/scripts/test/test_mv.json +6 -4
- package/scripts/test/test_replicate.json +18 -0
- package/scripts/test/test_vision.json +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"title": "Documentary",
|
|
3
|
+
"description": "Template for documentary, biography, and other long form content.",
|
|
4
|
+
"systemPrompt": "Generate 30 to 40 beats. Another AI will generate image for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
|
|
5
|
+
"presentationStyle": {
|
|
6
|
+
"$mulmocast": {
|
|
7
|
+
"version": "1.1",
|
|
8
|
+
"credit": "closing"
|
|
9
|
+
},
|
|
10
|
+
"imageParams": {
|
|
11
|
+
"style": "<style>Photo realistic and cinematic. Let the art convey the story and emotions without text. Use the image for the aspect ratio</style>"
|
|
12
|
+
},
|
|
13
|
+
"canvasSize": {
|
|
14
|
+
"width": 1536,
|
|
15
|
+
"height": 1024
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"scriptName": "text_only_template.json"
|
|
19
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension, MulmoImageParams } from "../types/index.js";
|
|
1
|
+
import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension, MulmoImageParams, MulmoMovieParams } from "../types/index.js";
|
|
2
2
|
export declare const imagePreprocessAgent: (namedInputs: {
|
|
3
3
|
context: MulmoStudioContext;
|
|
4
4
|
beat: MulmoBeat;
|
|
@@ -31,6 +31,11 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
31
31
|
duration?: number;
|
|
32
32
|
audioFile?: string;
|
|
33
33
|
beatDuration?: number;
|
|
34
|
+
movieAgentInfo?: {
|
|
35
|
+
agent: string;
|
|
36
|
+
movieParams: MulmoMovieParams;
|
|
37
|
+
};
|
|
38
|
+
markdown?: string;
|
|
34
39
|
htmlPrompt?: undefined;
|
|
35
40
|
htmlImageFile?: undefined;
|
|
36
41
|
htmlPath?: undefined;
|
|
@@ -38,21 +43,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
38
43
|
} | {
|
|
39
44
|
imagePath: string;
|
|
40
45
|
imageFromMovie: boolean;
|
|
41
|
-
movieAgentInfo: {
|
|
42
|
-
agent: string;
|
|
43
|
-
movieParams: {
|
|
44
|
-
speed?: number | undefined;
|
|
45
|
-
provider?: string | undefined;
|
|
46
|
-
model?: string | undefined;
|
|
47
|
-
fillOption?: {
|
|
48
|
-
style: "aspectFit" | "aspectFill";
|
|
49
|
-
} | undefined;
|
|
50
|
-
transition?: {
|
|
51
|
-
type: "fade" | "slideout_left";
|
|
52
|
-
duration: number;
|
|
53
|
-
} | undefined;
|
|
54
|
-
};
|
|
55
|
-
};
|
|
56
46
|
imageParams: MulmoImageParams;
|
|
57
47
|
movieFile: string | undefined;
|
|
58
48
|
soundEffectFile?: string;
|
|
@@ -71,6 +61,11 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
71
61
|
duration?: number;
|
|
72
62
|
audioFile?: string;
|
|
73
63
|
beatDuration?: number;
|
|
64
|
+
movieAgentInfo?: {
|
|
65
|
+
agent: string;
|
|
66
|
+
movieParams: MulmoMovieParams;
|
|
67
|
+
};
|
|
68
|
+
markdown?: string;
|
|
74
69
|
htmlPrompt?: undefined;
|
|
75
70
|
htmlImageFile?: undefined;
|
|
76
71
|
htmlPath?: undefined;
|
|
@@ -81,21 +76,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
81
76
|
imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
|
|
82
77
|
prompt: string;
|
|
83
78
|
referenceImages: string[];
|
|
84
|
-
movieAgentInfo: {
|
|
85
|
-
agent: string;
|
|
86
|
-
movieParams: {
|
|
87
|
-
speed?: number | undefined;
|
|
88
|
-
provider?: string | undefined;
|
|
89
|
-
model?: string | undefined;
|
|
90
|
-
fillOption?: {
|
|
91
|
-
style: "aspectFit" | "aspectFill";
|
|
92
|
-
} | undefined;
|
|
93
|
-
transition?: {
|
|
94
|
-
type: "fade" | "slideout_left";
|
|
95
|
-
duration: number;
|
|
96
|
-
} | undefined;
|
|
97
|
-
};
|
|
98
|
-
};
|
|
99
79
|
imageParams: MulmoImageParams;
|
|
100
80
|
movieFile: string | undefined;
|
|
101
81
|
soundEffectFile?: string;
|
|
@@ -114,6 +94,11 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
114
94
|
duration?: number;
|
|
115
95
|
audioFile?: string;
|
|
116
96
|
beatDuration?: number;
|
|
97
|
+
movieAgentInfo?: {
|
|
98
|
+
agent: string;
|
|
99
|
+
movieParams: MulmoMovieParams;
|
|
100
|
+
};
|
|
101
|
+
markdown?: string;
|
|
117
102
|
htmlPrompt?: undefined;
|
|
118
103
|
htmlImageFile?: undefined;
|
|
119
104
|
htmlPath?: undefined;
|
|
@@ -2,7 +2,6 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMeth
|
|
|
2
2
|
import { getBeatPngImagePath, getBeatMoviePaths, getAudioFilePath } from "../utils/file.js";
|
|
3
3
|
import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
|
|
4
4
|
import { renderHTMLToImage } from "../utils/markdown.js";
|
|
5
|
-
import { GraphAILogger } from "graphai";
|
|
6
5
|
import { beatId } from "../utils/utils.js";
|
|
7
6
|
const htmlStyle = (context, beat) => {
|
|
8
7
|
return {
|
|
@@ -57,21 +56,23 @@ export const imagePreprocessAgent = async (namedInputs) => {
|
|
|
57
56
|
returnValue.audioFile = studioBeat?.audioFile;
|
|
58
57
|
}
|
|
59
58
|
}
|
|
59
|
+
returnValue.movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
|
|
60
60
|
if (beat.image) {
|
|
61
61
|
const plugin = MulmoBeatMethods.getPlugin(beat);
|
|
62
62
|
const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
|
|
63
|
+
if (plugin.markdown) {
|
|
64
|
+
returnValue.markdown = plugin.markdown({ beat, context, imagePath, ...htmlStyle(context, beat) });
|
|
65
|
+
}
|
|
63
66
|
// undefined prompt indicates that image generation is not needed
|
|
64
67
|
return { ...returnValue, imagePath: pluginPath, referenceImageForMovie: pluginPath };
|
|
65
68
|
}
|
|
66
|
-
const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
|
|
67
|
-
GraphAILogger.log(`movieParams: ${index}`, movieAgentInfo.movieParams, returnValue.soundEffectAgentInfo, "\n", beat.moviePrompt, beat.soundEffectPrompt);
|
|
68
69
|
if (beat.moviePrompt && !beat.imagePrompt) {
|
|
69
|
-
return { ...returnValue, imagePath, imageFromMovie: true
|
|
70
|
+
return { ...returnValue, imagePath, imageFromMovie: true }; // no image prompt, only movie prompt
|
|
70
71
|
}
|
|
71
72
|
// referenceImages for "edit_image", openai agent.
|
|
72
73
|
const referenceImages = MulmoBeatMethods.getImageReferenceForImageGenerator(beat, imageRefs);
|
|
73
74
|
const prompt = imagePrompt(beat, imageAgentInfo.imageParams.style);
|
|
74
|
-
return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages
|
|
75
|
+
return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages };
|
|
75
76
|
};
|
|
76
77
|
export const imagePluginAgent = async (namedInputs) => {
|
|
77
78
|
const { context, beat, index } = namedInputs;
|
|
@@ -4,7 +4,7 @@ import { getReferenceImagePath, resolveAssetPath } from "../utils/file.js";
|
|
|
4
4
|
import { getExtention } from "../utils/utils.js";
|
|
5
5
|
import { graphOption } from "./images.js";
|
|
6
6
|
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
7
|
-
import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent } from "../agents/index.js";
|
|
7
|
+
import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent } from "../agents/index.js";
|
|
8
8
|
// public api
|
|
9
9
|
// Application may call this function directly to generate reference image.
|
|
10
10
|
export const generateReferenceImage = async (inputs) => {
|
|
@@ -39,7 +39,7 @@ export const generateReferenceImage = async (inputs) => {
|
|
|
39
39
|
},
|
|
40
40
|
};
|
|
41
41
|
const options = await graphOption(context);
|
|
42
|
-
const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent }, options);
|
|
42
|
+
const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent, imageReplicateAgent }, options);
|
|
43
43
|
await graph.run();
|
|
44
44
|
return imagePath;
|
|
45
45
|
};
|
package/lib/actions/images.d.ts
CHANGED
|
@@ -54,6 +54,11 @@ export declare const beat_graph_data: {
|
|
|
54
54
|
duration?: number;
|
|
55
55
|
audioFile?: string;
|
|
56
56
|
beatDuration?: number;
|
|
57
|
+
movieAgentInfo?: {
|
|
58
|
+
agent: string;
|
|
59
|
+
movieParams: import("../types/type.js").MulmoMovieParams;
|
|
60
|
+
};
|
|
61
|
+
markdown?: string;
|
|
57
62
|
htmlPrompt?: undefined;
|
|
58
63
|
htmlImageFile?: undefined;
|
|
59
64
|
htmlPath?: undefined;
|
|
@@ -61,21 +66,6 @@ export declare const beat_graph_data: {
|
|
|
61
66
|
} | {
|
|
62
67
|
imagePath: string;
|
|
63
68
|
imageFromMovie: boolean;
|
|
64
|
-
movieAgentInfo: {
|
|
65
|
-
agent: string;
|
|
66
|
-
movieParams: {
|
|
67
|
-
speed?: number | undefined;
|
|
68
|
-
provider?: string | undefined;
|
|
69
|
-
model?: string | undefined;
|
|
70
|
-
fillOption?: {
|
|
71
|
-
style: "aspectFit" | "aspectFill";
|
|
72
|
-
} | undefined;
|
|
73
|
-
transition?: {
|
|
74
|
-
type: "fade" | "slideout_left";
|
|
75
|
-
duration: number;
|
|
76
|
-
} | undefined;
|
|
77
|
-
};
|
|
78
|
-
};
|
|
79
69
|
imageParams: MulmoImageParams;
|
|
80
70
|
movieFile: string | undefined;
|
|
81
71
|
soundEffectFile?: string;
|
|
@@ -94,6 +84,11 @@ export declare const beat_graph_data: {
|
|
|
94
84
|
duration?: number;
|
|
95
85
|
audioFile?: string;
|
|
96
86
|
beatDuration?: number;
|
|
87
|
+
movieAgentInfo?: {
|
|
88
|
+
agent: string;
|
|
89
|
+
movieParams: import("../types/type.js").MulmoMovieParams;
|
|
90
|
+
};
|
|
91
|
+
markdown?: string;
|
|
97
92
|
htmlPrompt?: undefined;
|
|
98
93
|
htmlImageFile?: undefined;
|
|
99
94
|
htmlPath?: undefined;
|
|
@@ -104,21 +99,6 @@ export declare const beat_graph_data: {
|
|
|
104
99
|
imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
|
|
105
100
|
prompt: string;
|
|
106
101
|
referenceImages: string[];
|
|
107
|
-
movieAgentInfo: {
|
|
108
|
-
agent: string;
|
|
109
|
-
movieParams: {
|
|
110
|
-
speed?: number | undefined;
|
|
111
|
-
provider?: string | undefined;
|
|
112
|
-
model?: string | undefined;
|
|
113
|
-
fillOption?: {
|
|
114
|
-
style: "aspectFit" | "aspectFill";
|
|
115
|
-
} | undefined;
|
|
116
|
-
transition?: {
|
|
117
|
-
type: "fade" | "slideout_left";
|
|
118
|
-
duration: number;
|
|
119
|
-
} | undefined;
|
|
120
|
-
};
|
|
121
|
-
};
|
|
122
102
|
imageParams: MulmoImageParams;
|
|
123
103
|
movieFile: string | undefined;
|
|
124
104
|
soundEffectFile?: string;
|
|
@@ -137,6 +117,11 @@ export declare const beat_graph_data: {
|
|
|
137
117
|
duration?: number;
|
|
138
118
|
audioFile?: string;
|
|
139
119
|
beatDuration?: number;
|
|
120
|
+
movieAgentInfo?: {
|
|
121
|
+
agent: string;
|
|
122
|
+
movieParams: import("../types/type.js").MulmoMovieParams;
|
|
123
|
+
};
|
|
124
|
+
markdown?: string;
|
|
140
125
|
htmlPrompt?: undefined;
|
|
141
126
|
htmlImageFile?: undefined;
|
|
142
127
|
htmlPath?: undefined;
|
|
@@ -377,6 +362,7 @@ export declare const beat_graph_data: {
|
|
|
377
362
|
lipSyncFile: string;
|
|
378
363
|
hasMovieAudio: string;
|
|
379
364
|
htmlImageFile: string;
|
|
365
|
+
markdown: string;
|
|
380
366
|
};
|
|
381
367
|
output: {
|
|
382
368
|
imageFile: string;
|
|
@@ -385,6 +371,7 @@ export declare const beat_graph_data: {
|
|
|
385
371
|
lipSyncFile: string;
|
|
386
372
|
hasMovieAudio: string;
|
|
387
373
|
htmlImageFile: string;
|
|
374
|
+
markdown: string;
|
|
388
375
|
};
|
|
389
376
|
isResult: boolean;
|
|
390
377
|
};
|
package/lib/actions/images.js
CHANGED
|
@@ -5,7 +5,7 @@ import * as vanilla from "@graphai/vanilla";
|
|
|
5
5
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
6
6
|
import { anthropicAgent } from "@graphai/anthropic_agent";
|
|
7
7
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
8
|
-
import { imageGenAIAgent, imageOpenaiAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
|
|
8
|
+
import { imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
|
|
9
9
|
import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
|
|
10
10
|
import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
|
|
11
11
|
import { fileCacheAgentFilter } from "../utils/filters.js";
|
|
@@ -17,6 +17,7 @@ const vanillaAgents = vanilla.default ?? vanilla;
|
|
|
17
17
|
const imageAgents = {
|
|
18
18
|
imageGenAIAgent,
|
|
19
19
|
imageOpenaiAgent,
|
|
20
|
+
imageReplicateAgent,
|
|
20
21
|
};
|
|
21
22
|
const movieAgents = {
|
|
22
23
|
movieGenAIAgent,
|
|
@@ -284,6 +285,7 @@ export const beat_graph_data = {
|
|
|
284
285
|
lipSyncFile: ":preprocessor.lipSyncFile",
|
|
285
286
|
hasMovieAudio: ":audioChecker.hasMovieAudio",
|
|
286
287
|
htmlImageFile: ":preprocessor.htmlImageFile",
|
|
288
|
+
markdown: ":preprocessor.markdown",
|
|
287
289
|
},
|
|
288
290
|
output: {
|
|
289
291
|
imageFile: ".imageFile",
|
|
@@ -292,6 +294,7 @@ export const beat_graph_data = {
|
|
|
292
294
|
lipSyncFile: ".lipSyncFile",
|
|
293
295
|
hasMovieAudio: ".hasMovieAudio",
|
|
294
296
|
htmlImageFile: ".htmlImageFile",
|
|
297
|
+
markdown: ".markdown",
|
|
295
298
|
},
|
|
296
299
|
isResult: true,
|
|
297
300
|
},
|
package/lib/actions/index.d.ts
CHANGED
package/lib/actions/index.js
CHANGED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import { localizedText } from "../utils/utils.js";
|
|
3
|
+
import { writingMessage } from "../utils/file.js";
|
|
4
|
+
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
5
|
+
import path from "path";
|
|
6
|
+
const generateMarkdownContent = (context, imageWidth) => {
|
|
7
|
+
const { studio, multiLingual, lang = "en" } = context;
|
|
8
|
+
const title = studio.script.title || "MulmoCast Content";
|
|
9
|
+
const description = studio.script.description || "";
|
|
10
|
+
let markdown = `# ${title}\n\n`;
|
|
11
|
+
if (description) {
|
|
12
|
+
markdown += `${description}\n\n`;
|
|
13
|
+
}
|
|
14
|
+
studio.script.beats.forEach((beat, index) => {
|
|
15
|
+
const text = localizedText(beat, multiLingual?.[index], lang);
|
|
16
|
+
const studioBeat = studio.beats[index];
|
|
17
|
+
if (text.trim() || studioBeat?.markdown || studioBeat?.imageFile) {
|
|
18
|
+
if (studioBeat?.markdown) {
|
|
19
|
+
markdown += `${studioBeat.markdown}\n\n`;
|
|
20
|
+
}
|
|
21
|
+
else if (studioBeat?.imageFile && studioBeat.markdown !== "") {
|
|
22
|
+
const imagePath = path.relative(context.fileDirs.outDirPath, studioBeat.imageFile);
|
|
23
|
+
if (imageWidth) {
|
|
24
|
+
// Use HTML img tag for width control
|
|
25
|
+
const altText = `Beat ${index + 1}`;
|
|
26
|
+
markdown += `<img src="${imagePath}" alt="${altText}" width="${imageWidth}" />\n\n`;
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
// Use standard markdown image syntax
|
|
30
|
+
markdown += `\n\n`;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (text.trim()) {
|
|
34
|
+
markdown += `${text}\n\n`;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
return markdown;
|
|
39
|
+
};
|
|
40
|
+
export const markdownFilePath = (context) => {
|
|
41
|
+
const { studio, fileDirs, lang = "en" } = context;
|
|
42
|
+
// Add language suffix only when target language is different from script's original language
|
|
43
|
+
const langSuffix = studio.script.lang !== lang ? `_${lang}` : "";
|
|
44
|
+
const filename = `${studio.filename}${langSuffix}.md`;
|
|
45
|
+
return path.join(fileDirs.outDirPath, filename);
|
|
46
|
+
};
|
|
47
|
+
const generateMarkdown = async (context, imageWidth) => {
|
|
48
|
+
const outputMarkdownPath = markdownFilePath(context);
|
|
49
|
+
const markdownContent = generateMarkdownContent(context, imageWidth);
|
|
50
|
+
fs.writeFileSync(outputMarkdownPath, markdownContent, "utf8");
|
|
51
|
+
writingMessage(outputMarkdownPath);
|
|
52
|
+
};
|
|
53
|
+
export const markdown = async (context, imageWidth) => {
|
|
54
|
+
try {
|
|
55
|
+
MulmoStudioContextMethods.setSessionState(context, "markdown", true);
|
|
56
|
+
await generateMarkdown(context, imageWidth);
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
MulmoStudioContextMethods.setSessionState(context, "markdown", false);
|
|
60
|
+
}
|
|
61
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
|
+
import type { ReplicateImageAgentParams } from "../types/agent.js";
|
|
3
|
+
import type { AgentBufferResult, ImageAgentInputs, AgentConfig } from "../types/agent.js";
|
|
4
|
+
export type ReplicateImageAgentConfig = AgentConfig;
|
|
5
|
+
export declare const imageReplicateAgent: AgentFunction<ReplicateImageAgentParams, AgentBufferResult, ImageAgentInputs, ReplicateImageAgentConfig>;
|
|
6
|
+
declare const imageReplicateAgentInfo: AgentFunctionInfo;
|
|
7
|
+
export default imageReplicateAgentInfo;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { readFileSync } from "fs";
|
|
2
|
+
import { GraphAILogger } from "graphai";
|
|
3
|
+
import Replicate from "replicate";
|
|
4
|
+
import { getAspectRatio } from "./movie_replicate_agent.js";
|
|
5
|
+
import { provider2ImageAgent } from "../utils/provider2agent.js";
|
|
6
|
+
export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
7
|
+
const { prompt, referenceImages } = namedInputs;
|
|
8
|
+
const { canvasSize } = params;
|
|
9
|
+
const model = params.model ?? provider2ImageAgent.replicate.defaultModel;
|
|
10
|
+
const apiKey = config?.apiKey;
|
|
11
|
+
if (!apiKey) {
|
|
12
|
+
throw new Error("Replicate API key is required (REPLICATE_API_TOKEN)");
|
|
13
|
+
}
|
|
14
|
+
const replicate = new Replicate({
|
|
15
|
+
auth: apiKey,
|
|
16
|
+
});
|
|
17
|
+
const input = {
|
|
18
|
+
prompt,
|
|
19
|
+
aspect_ratio: getAspectRatio(canvasSize),
|
|
20
|
+
};
|
|
21
|
+
if (referenceImages && referenceImages.length > 0) {
|
|
22
|
+
input.image_input = referenceImages.map((image) => {
|
|
23
|
+
const buffer = readFileSync(image);
|
|
24
|
+
return `data:image/png;base64,${buffer.toString("base64")}`;
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
try {
|
|
28
|
+
const output = await replicate.run(model, { input });
|
|
29
|
+
// Download the generated video
|
|
30
|
+
if (output && Array.isArray(output) && output.length > 0 && typeof output[0] === "object" && "url" in output[0]) {
|
|
31
|
+
const imageUrl = output[0].url();
|
|
32
|
+
const imageResponse = await fetch(imageUrl);
|
|
33
|
+
if (!imageResponse.ok) {
|
|
34
|
+
throw new Error(`Error downloading video: ${imageResponse.status} - ${imageResponse.statusText}`);
|
|
35
|
+
}
|
|
36
|
+
const arrayBuffer = await imageResponse.arrayBuffer();
|
|
37
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
38
|
+
return { buffer };
|
|
39
|
+
}
|
|
40
|
+
throw new Error("ERROR: generateImage returned undefined");
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
GraphAILogger.info("Replicate generation error:", error);
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
const imageReplicateAgentInfo = {
|
|
48
|
+
name: "imageReplicateAgent",
|
|
49
|
+
agent: imageReplicateAgent,
|
|
50
|
+
mock: imageReplicateAgent,
|
|
51
|
+
samples: [],
|
|
52
|
+
description: "Replicate Image agent using FLUX and other models",
|
|
53
|
+
category: ["image"],
|
|
54
|
+
author: "Receptron Team",
|
|
55
|
+
repository: "https://github.com/receptron/mulmocast-cli/",
|
|
56
|
+
license: "MIT",
|
|
57
|
+
environmentVariables: ["REPLICATE_API_TOKEN"],
|
|
58
|
+
};
|
|
59
|
+
export default imageReplicateAgentInfo;
|
package/lib/agents/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
|
|
|
2
2
|
import combineAudioFilesAgent from "./combine_audio_files_agent.js";
|
|
3
3
|
import imageGenAIAgent from "./image_genai_agent.js";
|
|
4
4
|
import imageOpenaiAgent from "./image_openai_agent.js";
|
|
5
|
+
import imageReplicateAgent from "./image_replicate_agent.js";
|
|
5
6
|
import tavilySearchAgent from "./tavily_agent.js";
|
|
6
7
|
import movieGenAIAgent from "./movie_genai_agent.js";
|
|
7
8
|
import movieReplicateAgent from "./movie_replicate_agent.js";
|
|
@@ -18,4 +19,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
|
|
|
18
19
|
import { textInputAgent } from "@graphai/input_agents";
|
|
19
20
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
20
21
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
21
|
-
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
|
|
22
|
+
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
|
package/lib/agents/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
|
|
|
2
2
|
import combineAudioFilesAgent from "./combine_audio_files_agent.js";
|
|
3
3
|
import imageGenAIAgent from "./image_genai_agent.js";
|
|
4
4
|
import imageOpenaiAgent from "./image_openai_agent.js";
|
|
5
|
+
import imageReplicateAgent from "./image_replicate_agent.js";
|
|
5
6
|
import tavilySearchAgent from "./tavily_agent.js";
|
|
6
7
|
import movieGenAIAgent from "./movie_genai_agent.js";
|
|
7
8
|
import movieReplicateAgent from "./movie_replicate_agent.js";
|
|
@@ -19,4 +20,4 @@ import { textInputAgent } from "@graphai/input_agents";
|
|
|
19
20
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
20
21
|
// import * as vanilla from "@graphai/vanilla";
|
|
21
22
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
22
|
-
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
|
|
23
|
+
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
|
|
@@ -54,15 +54,17 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
|
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
export const getAspectRatio = (canvasSize) => {
|
|
57
|
-
|
|
57
|
+
const ratio = canvasSize.width / canvasSize.height;
|
|
58
|
+
const tolerance = 0.1;
|
|
59
|
+
if (ratio > 4 / 3 + tolerance)
|
|
58
60
|
return "16:9";
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
else {
|
|
61
|
+
if (ratio > 4 / 3 - tolerance)
|
|
62
|
+
return "4:3";
|
|
63
|
+
if (ratio > 3 / 4 + tolerance)
|
|
64
64
|
return "1:1";
|
|
65
|
-
|
|
65
|
+
if (ratio > 3 / 4 - tolerance)
|
|
66
|
+
return "3:4";
|
|
67
|
+
return "9:16";
|
|
66
68
|
};
|
|
67
69
|
export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
68
70
|
const { prompt, imagePath } = namedInputs;
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
import { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
|
-
type Article = {
|
|
3
|
-
url: string;
|
|
4
|
-
title: string | null;
|
|
5
|
-
byline: string | null;
|
|
6
|
-
excerpt: string | null;
|
|
7
|
-
length: number | null;
|
|
8
|
-
textContent: string | null;
|
|
9
|
-
};
|
|
10
|
-
export declare const fetchArticle: (url: string) => Promise<Article>;
|
|
11
2
|
export declare const puppeteerCrawlerAgent: AgentFunction;
|
|
12
3
|
declare const puppeteerCrawlerAgentInfo: AgentFunctionInfo;
|
|
13
4
|
export default puppeteerCrawlerAgentInfo;
|
|
@@ -17,7 +17,7 @@ const waitStable = async (page, ms = 1200, step = 200) => {
|
|
|
17
17
|
await new Promise((r) => setTimeout(r, step));
|
|
18
18
|
}
|
|
19
19
|
};
|
|
20
|
-
|
|
20
|
+
const fetchArticle = async (url) => {
|
|
21
21
|
const browser = await puppeteer.launch({
|
|
22
22
|
headless: true,
|
|
23
23
|
args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"],
|
package/lib/cli/bin.js
CHANGED
|
@@ -10,6 +10,7 @@ import * as audioCmd from "./commands/audio/index.js";
|
|
|
10
10
|
import * as imagesCmd from "./commands/image/index.js";
|
|
11
11
|
import * as movieCmd from "./commands/movie/index.js";
|
|
12
12
|
import * as pdfCmd from "./commands/pdf/index.js";
|
|
13
|
+
import * as markdownCmd from "./commands/markdown/index.js";
|
|
13
14
|
import * as toolCmd from "./commands/tool/index.js";
|
|
14
15
|
import { GraphAILogger } from "graphai";
|
|
15
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -32,6 +33,7 @@ export const main = async () => {
|
|
|
32
33
|
.command(imagesCmd)
|
|
33
34
|
.command(movieCmd)
|
|
34
35
|
.command(pdfCmd)
|
|
36
|
+
.command(markdownCmd)
|
|
35
37
|
.command(toolCmd)
|
|
36
38
|
.demandCommand()
|
|
37
39
|
.strict()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Argv } from "yargs";
|
|
2
|
+
export declare const builder: (yargs: Argv) => Argv<{
|
|
3
|
+
o: string | undefined;
|
|
4
|
+
} & {
|
|
5
|
+
b: string | undefined;
|
|
6
|
+
} & {
|
|
7
|
+
l: string | undefined;
|
|
8
|
+
} & {
|
|
9
|
+
f: boolean;
|
|
10
|
+
} & {
|
|
11
|
+
p: string | undefined;
|
|
12
|
+
} & {
|
|
13
|
+
file: string | undefined;
|
|
14
|
+
} & {
|
|
15
|
+
image_width: string | undefined;
|
|
16
|
+
}>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { images, markdown } from "../../../actions/index.js";
|
|
2
|
+
import { initializeContext, runTranslateIfNeeded } from "../../helpers.js";
|
|
3
|
+
export const handler = async (argv) => {
|
|
4
|
+
const context = await initializeContext(argv);
|
|
5
|
+
if (!context) {
|
|
6
|
+
process.exit(1);
|
|
7
|
+
}
|
|
8
|
+
await runTranslateIfNeeded(context);
|
|
9
|
+
await images(context);
|
|
10
|
+
await markdown(context, argv.image_width);
|
|
11
|
+
};
|
|
@@ -256,6 +256,48 @@ export const promptTemplates = [
|
|
|
256
256
|
systemPrompt: "Another AI will generate comic strips for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
|
|
257
257
|
title: "American Comic Strips",
|
|
258
258
|
},
|
|
259
|
+
{
|
|
260
|
+
description: "Template for documentary, biography, and other long form content.",
|
|
261
|
+
filename: "documentary",
|
|
262
|
+
presentationStyle: {
|
|
263
|
+
$mulmocast: {
|
|
264
|
+
credit: "closing",
|
|
265
|
+
version: "1.1",
|
|
266
|
+
},
|
|
267
|
+
audioParams: {
|
|
268
|
+
audioVolume: 1,
|
|
269
|
+
bgmVolume: 0.2,
|
|
270
|
+
closingPadding: 0.8,
|
|
271
|
+
introPadding: 1,
|
|
272
|
+
outroPadding: 1,
|
|
273
|
+
padding: 0.3,
|
|
274
|
+
suppressSpeech: false,
|
|
275
|
+
},
|
|
276
|
+
canvasSize: {
|
|
277
|
+
height: 1024,
|
|
278
|
+
width: 1536,
|
|
279
|
+
},
|
|
280
|
+
imageParams: {
|
|
281
|
+
style: "<style>Photo realistic and cinematic. Let the art convey the story and emotions without text. Use the image for the aspect ratio</style>",
|
|
282
|
+
},
|
|
283
|
+
soundEffectParams: {
|
|
284
|
+
provider: "replicate",
|
|
285
|
+
},
|
|
286
|
+
speechParams: {
|
|
287
|
+
speakers: {
|
|
288
|
+
Presenter: {
|
|
289
|
+
displayName: {
|
|
290
|
+
en: "Presenter",
|
|
291
|
+
},
|
|
292
|
+
voiceId: "shimmer",
|
|
293
|
+
},
|
|
294
|
+
},
|
|
295
|
+
},
|
|
296
|
+
},
|
|
297
|
+
scriptName: "text_only_template.json",
|
|
298
|
+
systemPrompt: "Generate 30 to 40 beats. Another AI will generate image for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
|
|
299
|
+
title: "Documentary",
|
|
300
|
+
},
|
|
259
301
|
{
|
|
260
302
|
description: "Template for Dr. Slump style comic presentation.",
|
|
261
303
|
filename: "drslump_comic",
|