mulmocast 0.0.13 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/assets/html/pdf_handout.html +85 -0
- package/assets/html/pdf_slide.html +55 -0
- package/assets/html/pdf_talk.html +76 -0
- package/lib/actions/audio.d.ts +1 -0
- package/lib/actions/audio.js +8 -2
- package/lib/actions/images.d.ts +85 -1
- package/lib/actions/images.js +130 -88
- package/lib/actions/movie.d.ts +1 -0
- package/lib/actions/movie.js +14 -3
- package/lib/actions/pdf.d.ts +1 -0
- package/lib/actions/pdf.js +132 -202
- package/lib/agents/combine_audio_files_agent.js +10 -8
- package/lib/agents/image_mock_agent.d.ts +4 -0
- package/lib/agents/image_mock_agent.js +18 -0
- package/lib/agents/index.d.ts +3 -1
- package/lib/agents/index.js +3 -1
- package/lib/agents/media_mock_agent.d.ts +4 -0
- package/lib/agents/media_mock_agent.js +18 -0
- package/lib/agents/tts_openai_agent.js +9 -1
- package/lib/cli/commands/audio/builder.d.ts +2 -0
- package/lib/cli/commands/image/builder.d.ts +2 -0
- package/lib/cli/commands/movie/builder.d.ts +2 -0
- package/lib/cli/commands/pdf/builder.d.ts +2 -0
- package/lib/cli/commands/translate/builder.d.ts +2 -0
- package/lib/cli/common.d.ts +2 -0
- package/lib/cli/common.js +5 -0
- package/lib/cli/helpers.js +1 -0
- package/lib/methods/mulmo_script.d.ts +1 -1
- package/lib/methods/mulmo_script.js +2 -2
- package/lib/types/type.d.ts +1 -0
- package/lib/utils/filters.js +1 -0
- package/lib/utils/markdown.js +1 -1
- package/package.json +7 -8
- package/assets/font/NotoSansJP-Regular.ttf +0 -0
package/README.md
CHANGED
|
@@ -103,9 +103,13 @@ GOOGLE_PROJECT_ID=your_google_project_id
|
|
|
103
103
|
|
|
104
104
|
See also [pre-requisites for Google's image generation model](./docs/pre-requisites-google.md)
|
|
105
105
|
|
|
106
|
-
#### (Optional) For
|
|
106
|
+
#### (Optional) For TTS models
|
|
107
107
|
```bash
|
|
108
|
+
# For Nijivoice TTS
|
|
108
109
|
NIJIVOICE_API_KEY=your_nijivoice_api_key
|
|
110
|
+
|
|
111
|
+
# For ElevenLabs TTS
|
|
112
|
+
ELEVENLABS_API_KEY=your_elevenlabs_api_key
|
|
109
113
|
```
|
|
110
114
|
|
|
111
115
|
#### (Optional) to access web in mulmo tool
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="${lang}">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>${title}</title>
|
|
7
|
+
<style>
|
|
8
|
+
@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+JP:wght@400;700&display=swap');
|
|
9
|
+
|
|
10
|
+
* {
|
|
11
|
+
margin: 0;
|
|
12
|
+
padding: 0;
|
|
13
|
+
box-sizing: border-box;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
body {
|
|
17
|
+
font-family: 'Noto Sans JP', sans-serif;
|
|
18
|
+
font-size: 16px;
|
|
19
|
+
line-height: 1.6;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #fff;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
@page {
|
|
25
|
+
size: ${page_size};
|
|
26
|
+
margin: 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
.page {
|
|
30
|
+
page-break-after: always;
|
|
31
|
+
width: 100%;
|
|
32
|
+
height: 100vh;
|
|
33
|
+
position: relative;
|
|
34
|
+
overflow: hidden;
|
|
35
|
+
padding: 15px;
|
|
36
|
+
display: ${page_layout};
|
|
37
|
+
${page_direction}
|
|
38
|
+
gap: 15px;
|
|
39
|
+
background: #fff;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
.page:last-child {
|
|
43
|
+
page-break-after: avoid;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
img {
|
|
47
|
+
max-width: 100%;
|
|
48
|
+
max-height: 100%;
|
|
49
|
+
object-fit: contain;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
.handout-item {
|
|
53
|
+
display: flex;
|
|
54
|
+
flex-direction: ${flex_direction};
|
|
55
|
+
border: 1px solid #ddd;
|
|
56
|
+
overflow: hidden;
|
|
57
|
+
${item_flex}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.handout-image {
|
|
61
|
+
${image_size}
|
|
62
|
+
display: flex;
|
|
63
|
+
align-items: center;
|
|
64
|
+
justify-content: center;
|
|
65
|
+
background: #f9f9f9;
|
|
66
|
+
padding: 5px;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
.handout-text {
|
|
70
|
+
${text_size}
|
|
71
|
+
padding: 8px;
|
|
72
|
+
font-size: 14px;
|
|
73
|
+
overflow: hidden;
|
|
74
|
+
background: #fff;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
.handout-text p {
|
|
78
|
+
margin: 0.3em 0;
|
|
79
|
+
}
|
|
80
|
+
</style>
|
|
81
|
+
</head>
|
|
82
|
+
<body>
|
|
83
|
+
${pages}
|
|
84
|
+
</body>
|
|
85
|
+
</html>
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="${lang}">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>${title}</title>
|
|
7
|
+
<style>
|
|
8
|
+
@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+JP:wght@400;700&display=swap');
|
|
9
|
+
|
|
10
|
+
* {
|
|
11
|
+
margin: 0;
|
|
12
|
+
padding: 0;
|
|
13
|
+
box-sizing: border-box;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
body {
|
|
17
|
+
font-family: 'Noto Sans JP', sans-serif;
|
|
18
|
+
font-size: 14px;
|
|
19
|
+
line-height: 1.6;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #fff;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
@page {
|
|
25
|
+
size: ${page_size};
|
|
26
|
+
margin: 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
.page {
|
|
30
|
+
page-break-after: always;
|
|
31
|
+
width: 100%;
|
|
32
|
+
height: 100vh;
|
|
33
|
+
position: relative;
|
|
34
|
+
overflow: hidden;
|
|
35
|
+
display: flex;
|
|
36
|
+
align-items: center;
|
|
37
|
+
justify-content: center;
|
|
38
|
+
background: #fff;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
.page:last-child {
|
|
42
|
+
page-break-after: avoid;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
img {
|
|
46
|
+
max-width: 100%;
|
|
47
|
+
max-height: 100%;
|
|
48
|
+
object-fit: contain;
|
|
49
|
+
}
|
|
50
|
+
</style>
|
|
51
|
+
</head>
|
|
52
|
+
<body>
|
|
53
|
+
${pages}
|
|
54
|
+
</body>
|
|
55
|
+
</html>
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="${lang}">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>${title}</title>
|
|
7
|
+
<style>
|
|
8
|
+
@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+JP:wght@400;700&display=swap');
|
|
9
|
+
|
|
10
|
+
* {
|
|
11
|
+
margin: 0;
|
|
12
|
+
padding: 0;
|
|
13
|
+
box-sizing: border-box;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
body {
|
|
17
|
+
font-family: 'Noto Sans JP', sans-serif;
|
|
18
|
+
font-size: 17px;
|
|
19
|
+
line-height: 1.4;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #fff;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
@page {
|
|
25
|
+
size: ${page_size};
|
|
26
|
+
margin: 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
.page {
|
|
30
|
+
page-break-after: always;
|
|
31
|
+
width: 100%;
|
|
32
|
+
height: 100vh;
|
|
33
|
+
position: relative;
|
|
34
|
+
overflow: hidden;
|
|
35
|
+
padding: 20px;
|
|
36
|
+
display: flex;
|
|
37
|
+
flex-direction: column;
|
|
38
|
+
background: #fff;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
.page:last-child {
|
|
42
|
+
page-break-after: avoid;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
img {
|
|
46
|
+
max-width: 100%;
|
|
47
|
+
max-height: 100%;
|
|
48
|
+
object-fit: contain;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
.image-container {
|
|
52
|
+
flex: 1;
|
|
53
|
+
display: flex;
|
|
54
|
+
align-items: center;
|
|
55
|
+
justify-content: center;
|
|
56
|
+
margin-bottom: 20px;
|
|
57
|
+
border: 1px solid #ddd;
|
|
58
|
+
background: #f9f9f9;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
.text-container {
|
|
62
|
+
padding: 10px;
|
|
63
|
+
background: #fff;
|
|
64
|
+
border-top: 2px solid #333;
|
|
65
|
+
min-height: 120px;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
.text-container p {
|
|
69
|
+
margin: 0.5em 0;
|
|
70
|
+
}
|
|
71
|
+
</style>
|
|
72
|
+
</head>
|
|
73
|
+
<body>
|
|
74
|
+
${pages}
|
|
75
|
+
</body>
|
|
76
|
+
</html>
|
package/lib/actions/audio.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
2
|
import type { CallbackFunction } from "graphai";
|
|
3
3
|
import { MulmoStudioContext } from "../types/index.js";
|
|
4
|
+
export declare const audioFilePath: (context: MulmoStudioContext) => string;
|
|
4
5
|
export declare const audio: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
|
package/lib/actions/audio.js
CHANGED
|
@@ -22,6 +22,7 @@ const provider_to_agent = {
|
|
|
22
22
|
openai: "ttsOpenaiAgent",
|
|
23
23
|
google: "ttsGoogleAgent",
|
|
24
24
|
elevenlabs: "ttsElevenlabsAgent",
|
|
25
|
+
mock: "mediaMockAgent",
|
|
25
26
|
};
|
|
26
27
|
const getAudioPath = (context, beat, audioFile, audioDirPath) => {
|
|
27
28
|
if (beat.audio?.type === "audio") {
|
|
@@ -31,7 +32,7 @@ const getAudioPath = (context, beat, audioFile, audioDirPath) => {
|
|
|
31
32
|
}
|
|
32
33
|
throw new Error("Invalid audio source");
|
|
33
34
|
}
|
|
34
|
-
if (beat.text === "") {
|
|
35
|
+
if (beat.text === undefined || beat.text === "") {
|
|
35
36
|
return undefined; // It indicates that the audio is not needed.
|
|
36
37
|
}
|
|
37
38
|
return getAudioSegmentFilePath(audioDirPath, context.studio.filename, audioFile);
|
|
@@ -166,12 +167,17 @@ const agentFilters = [
|
|
|
166
167
|
nodeIds: ["tts"],
|
|
167
168
|
},
|
|
168
169
|
];
|
|
170
|
+
export const audioFilePath = (context) => {
|
|
171
|
+
const { studio, fileDirs } = context;
|
|
172
|
+
const { outDirPath } = fileDirs;
|
|
173
|
+
return getAudioArtifactFilePath(outDirPath, studio.filename);
|
|
174
|
+
};
|
|
169
175
|
export const audio = async (context, callbacks) => {
|
|
170
176
|
try {
|
|
171
177
|
MulmoStudioContextMethods.setSessionState(context, "audio", true);
|
|
172
178
|
const { studio, fileDirs, lang } = context;
|
|
173
179
|
const { outDirPath, audioDirPath } = fileDirs;
|
|
174
|
-
const audioArtifactFilePath =
|
|
180
|
+
const audioArtifactFilePath = audioFilePath(context);
|
|
175
181
|
const audioSegmentDirPath = getAudioSegmentDirPath(audioDirPath, studio.filename);
|
|
176
182
|
const audioCombinedFilePath = getAudioCombinedFilePath(audioDirPath, studio.filename, lang);
|
|
177
183
|
const outputStudioFilePath = getOutputStudioFilePath(outDirPath, studio.filename);
|
package/lib/actions/images.d.ts
CHANGED
|
@@ -1,3 +1,87 @@
|
|
|
1
1
|
import type { CallbackFunction } from "graphai";
|
|
2
|
-
import { MulmoStudioContext } from "../types/index.js";
|
|
2
|
+
import { MulmoStudioContext, MulmoBeat, Text2ImageAgentInfo } from "../types/index.js";
|
|
3
|
+
export declare const imagePreprocessAgent: (namedInputs: {
|
|
4
|
+
context: MulmoStudioContext;
|
|
5
|
+
beat: MulmoBeat;
|
|
6
|
+
index: number;
|
|
7
|
+
suffix: string;
|
|
8
|
+
imageDirPath: string;
|
|
9
|
+
imageAgentInfo: Text2ImageAgentInfo;
|
|
10
|
+
imageRefs: Record<string, string>;
|
|
11
|
+
}) => Promise<{
|
|
12
|
+
imageParams: {
|
|
13
|
+
model?: string | undefined;
|
|
14
|
+
style?: string | undefined;
|
|
15
|
+
moderation?: string | undefined;
|
|
16
|
+
images?: Record<string, {
|
|
17
|
+
type: "image";
|
|
18
|
+
source: {
|
|
19
|
+
url: string;
|
|
20
|
+
kind: "url";
|
|
21
|
+
} | {
|
|
22
|
+
kind: "base64";
|
|
23
|
+
data: string;
|
|
24
|
+
} | {
|
|
25
|
+
text: string;
|
|
26
|
+
kind: "text";
|
|
27
|
+
} | {
|
|
28
|
+
path: string;
|
|
29
|
+
kind: "path";
|
|
30
|
+
};
|
|
31
|
+
}> | undefined;
|
|
32
|
+
};
|
|
33
|
+
movieFile: string | undefined;
|
|
34
|
+
imagePath: string | undefined;
|
|
35
|
+
} | {
|
|
36
|
+
images: string[];
|
|
37
|
+
imageParams: {
|
|
38
|
+
model?: string | undefined;
|
|
39
|
+
style?: string | undefined;
|
|
40
|
+
moderation?: string | undefined;
|
|
41
|
+
images?: Record<string, {
|
|
42
|
+
type: "image";
|
|
43
|
+
source: {
|
|
44
|
+
url: string;
|
|
45
|
+
kind: "url";
|
|
46
|
+
} | {
|
|
47
|
+
kind: "base64";
|
|
48
|
+
data: string;
|
|
49
|
+
} | {
|
|
50
|
+
text: string;
|
|
51
|
+
kind: "text";
|
|
52
|
+
} | {
|
|
53
|
+
path: string;
|
|
54
|
+
kind: "path";
|
|
55
|
+
};
|
|
56
|
+
}> | undefined;
|
|
57
|
+
};
|
|
58
|
+
movieFile: string | undefined;
|
|
59
|
+
} | {
|
|
60
|
+
images: string[];
|
|
61
|
+
imageParams: {
|
|
62
|
+
model?: string | undefined;
|
|
63
|
+
style?: string | undefined;
|
|
64
|
+
moderation?: string | undefined;
|
|
65
|
+
images?: Record<string, {
|
|
66
|
+
type: "image";
|
|
67
|
+
source: {
|
|
68
|
+
url: string;
|
|
69
|
+
kind: "url";
|
|
70
|
+
} | {
|
|
71
|
+
kind: "base64";
|
|
72
|
+
data: string;
|
|
73
|
+
} | {
|
|
74
|
+
text: string;
|
|
75
|
+
kind: "text";
|
|
76
|
+
} | {
|
|
77
|
+
path: string;
|
|
78
|
+
kind: "path";
|
|
79
|
+
};
|
|
80
|
+
}> | undefined;
|
|
81
|
+
};
|
|
82
|
+
movieFile: string | undefined;
|
|
83
|
+
imagePath: string;
|
|
84
|
+
prompt: string;
|
|
85
|
+
}>;
|
|
3
86
|
export declare const images: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
|
|
87
|
+
export declare const generateBeatImage: (index: number, context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
|
package/lib/actions/images.js
CHANGED
|
@@ -5,9 +5,7 @@ import * as agents from "@graphai/vanilla";
|
|
|
5
5
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
6
6
|
import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
|
|
7
7
|
import { fileCacheAgentFilter } from "../utils/filters.js";
|
|
8
|
-
import imageGoogleAgent from "../agents/
|
|
9
|
-
import imageOpenaiAgent from "../agents/image_openai_agent.js";
|
|
10
|
-
import movieGoogleAgent from "../agents/movie_google_agent.js";
|
|
8
|
+
import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent } from "../agents/index.js";
|
|
11
9
|
import { MulmoScriptMethods, MulmoStudioContextMethods } from "../methods/index.js";
|
|
12
10
|
import { imagePlugins } from "../utils/image_plugins/index.js";
|
|
13
11
|
import { imagePrompt } from "../utils/prompt.js";
|
|
@@ -21,7 +19,7 @@ const htmlStyle = (script, beat) => {
|
|
|
21
19
|
textSlideStyle: MulmoScriptMethods.getTextSlideStyle(script, beat),
|
|
22
20
|
};
|
|
23
21
|
};
|
|
24
|
-
const imagePreprocessAgent = async (namedInputs) => {
|
|
22
|
+
export const imagePreprocessAgent = async (namedInputs) => {
|
|
25
23
|
const { context, beat, index, suffix, imageDirPath, imageAgentInfo, imageRefs } = namedInputs;
|
|
26
24
|
const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
|
|
27
25
|
const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
|
|
@@ -56,6 +54,85 @@ const imagePreprocessAgent = async (namedInputs) => {
|
|
|
56
54
|
const prompt = imagePrompt(beat, imageParams.style);
|
|
57
55
|
return { imagePath, prompt, ...returnValue, images };
|
|
58
56
|
};
|
|
57
|
+
const beat_graph_data = {
|
|
58
|
+
version: 0.5,
|
|
59
|
+
concurrency: 4,
|
|
60
|
+
nodes: {
|
|
61
|
+
context: {},
|
|
62
|
+
imageDirPath: {},
|
|
63
|
+
imageAgentInfo: {},
|
|
64
|
+
movieAgentInfo: {},
|
|
65
|
+
imageRefs: {},
|
|
66
|
+
beat: {},
|
|
67
|
+
__mapIndex: {},
|
|
68
|
+
preprocessor: {
|
|
69
|
+
agent: imagePreprocessAgent,
|
|
70
|
+
inputs: {
|
|
71
|
+
context: ":context",
|
|
72
|
+
beat: ":beat",
|
|
73
|
+
index: ":__mapIndex",
|
|
74
|
+
suffix: "p",
|
|
75
|
+
imageDirPath: ":imageDirPath",
|
|
76
|
+
imageAgentInfo: ":imageAgentInfo",
|
|
77
|
+
imageRefs: ":imageRefs",
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
imageGenerator: {
|
|
81
|
+
if: ":preprocessor.prompt",
|
|
82
|
+
agent: ":imageAgentInfo.agent",
|
|
83
|
+
retry: 3,
|
|
84
|
+
inputs: {
|
|
85
|
+
prompt: ":preprocessor.prompt",
|
|
86
|
+
images: ":preprocessor.images",
|
|
87
|
+
file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
|
|
88
|
+
text: ":preprocessor.prompt", // only for fileCacheAgentFilter
|
|
89
|
+
force: ":context.force", // only for fileCacheAgentFilter
|
|
90
|
+
mulmoContext: ":context", // for fileCacheAgentFilter
|
|
91
|
+
index: ":__mapIndex", // for fileCacheAgentFilter
|
|
92
|
+
sessionType: "image", // for fileCacheAgentFilter
|
|
93
|
+
params: {
|
|
94
|
+
model: ":preprocessor.imageParams.model",
|
|
95
|
+
moderation: ":preprocessor.imageParams.moderation",
|
|
96
|
+
canvasSize: ":context.studio.script.canvasSize",
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
defaultValue: {},
|
|
100
|
+
},
|
|
101
|
+
movieGenerator: {
|
|
102
|
+
if: ":preprocessor.movieFile",
|
|
103
|
+
agent: ":movieAgentInfo.agent",
|
|
104
|
+
inputs: {
|
|
105
|
+
onComplete: ":imageGenerator", // to wait for imageGenerator to finish
|
|
106
|
+
prompt: ":beat.moviePrompt",
|
|
107
|
+
imagePath: ":preprocessor.imagePath",
|
|
108
|
+
file: ":preprocessor.movieFile",
|
|
109
|
+
studio: ":context.studio", // for cache
|
|
110
|
+
mulmoContext: ":context", // for fileCacheAgentFilter
|
|
111
|
+
index: ":__mapIndex", // for cache
|
|
112
|
+
sessionType: "movie", // for cache
|
|
113
|
+
params: {
|
|
114
|
+
model: ":context.studio.script.movieParams.model",
|
|
115
|
+
duration: ":beat.duration",
|
|
116
|
+
canvasSize: ":context.studio.script.canvasSize",
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
defaultValue: {},
|
|
120
|
+
},
|
|
121
|
+
output: {
|
|
122
|
+
agent: "copyAgent",
|
|
123
|
+
inputs: {
|
|
124
|
+
onComplete: ":movieGenerator", // to wait for movieGenerator to finish
|
|
125
|
+
imageFile: ":preprocessor.imagePath",
|
|
126
|
+
movieFile: ":preprocessor.movieFile",
|
|
127
|
+
},
|
|
128
|
+
output: {
|
|
129
|
+
imageFile: ".imageFile",
|
|
130
|
+
movieFile: ".movieFile",
|
|
131
|
+
},
|
|
132
|
+
isResult: true,
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
};
|
|
59
136
|
const graph_data = {
|
|
60
137
|
version: 0.5,
|
|
61
138
|
concurrency: 4,
|
|
@@ -63,6 +140,7 @@ const graph_data = {
|
|
|
63
140
|
context: {},
|
|
64
141
|
imageDirPath: {},
|
|
65
142
|
imageAgentInfo: {},
|
|
143
|
+
movieAgentInfo: {},
|
|
66
144
|
outputStudioFilePath: {},
|
|
67
145
|
imageRefs: {},
|
|
68
146
|
map: {
|
|
@@ -71,6 +149,7 @@ const graph_data = {
|
|
|
71
149
|
rows: ":context.studio.script.beats",
|
|
72
150
|
context: ":context",
|
|
73
151
|
imageAgentInfo: ":imageAgentInfo",
|
|
152
|
+
movieAgentInfo: ":movieAgentInfo",
|
|
74
153
|
imageDirPath: ":imageDirPath",
|
|
75
154
|
imageRefs: ":imageRefs",
|
|
76
155
|
},
|
|
@@ -79,80 +158,10 @@ const graph_data = {
|
|
|
79
158
|
rowKey: "beat",
|
|
80
159
|
compositeResult: true,
|
|
81
160
|
},
|
|
82
|
-
graph:
|
|
83
|
-
nodes: {
|
|
84
|
-
preprocessor: {
|
|
85
|
-
agent: imagePreprocessAgent,
|
|
86
|
-
inputs: {
|
|
87
|
-
context: ":context",
|
|
88
|
-
beat: ":beat",
|
|
89
|
-
index: ":__mapIndex",
|
|
90
|
-
suffix: "p",
|
|
91
|
-
imageDirPath: ":imageDirPath",
|
|
92
|
-
imageAgentInfo: ":imageAgentInfo",
|
|
93
|
-
imageRefs: ":imageRefs",
|
|
94
|
-
},
|
|
95
|
-
},
|
|
96
|
-
imageGenerator: {
|
|
97
|
-
if: ":preprocessor.prompt",
|
|
98
|
-
agent: ":imageAgentInfo.agent",
|
|
99
|
-
retry: 3,
|
|
100
|
-
inputs: {
|
|
101
|
-
prompt: ":preprocessor.prompt",
|
|
102
|
-
images: ":preprocessor.images",
|
|
103
|
-
file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
|
|
104
|
-
text: ":preprocessor.prompt", // only for fileCacheAgentFilter
|
|
105
|
-
force: ":context.force", // only for fileCacheAgentFilter
|
|
106
|
-
mulmoContext: ":context", // for fileCacheAgentFilter
|
|
107
|
-
index: ":__mapIndex", // for fileCacheAgentFilter
|
|
108
|
-
sessionType: "image", // for fileCacheAgentFilter
|
|
109
|
-
params: {
|
|
110
|
-
model: ":preprocessor.imageParams.model",
|
|
111
|
-
moderation: ":preprocessor.imageParams.moderation",
|
|
112
|
-
canvasSize: ":context.studio.script.canvasSize",
|
|
113
|
-
},
|
|
114
|
-
},
|
|
115
|
-
defaultValue: {},
|
|
116
|
-
},
|
|
117
|
-
movieGenerator: {
|
|
118
|
-
if: ":preprocessor.movieFile",
|
|
119
|
-
agent: "movieGoogleAgent",
|
|
120
|
-
inputs: {
|
|
121
|
-
onComplete: ":imageGenerator", // to wait for imageGenerator to finish
|
|
122
|
-
prompt: ":beat.moviePrompt",
|
|
123
|
-
imagePath: ":preprocessor.imagePath",
|
|
124
|
-
file: ":preprocessor.movieFile",
|
|
125
|
-
studio: ":context.studio", // for cache
|
|
126
|
-
index: ":__mapIndex", // for cache
|
|
127
|
-
sessionType: "movie", // for cache
|
|
128
|
-
params: {
|
|
129
|
-
model: ":context.studio.script.movieParams.model",
|
|
130
|
-
duration: ":beat.duration",
|
|
131
|
-
canvasSize: ":context.studio.script.canvasSize",
|
|
132
|
-
},
|
|
133
|
-
},
|
|
134
|
-
defaultValue: {},
|
|
135
|
-
},
|
|
136
|
-
onComplete: {
|
|
137
|
-
agent: "copyAgent",
|
|
138
|
-
inputs: {
|
|
139
|
-
onComplete: ":movieGenerator", // to wait for movieGenerator to finish
|
|
140
|
-
imageFile: ":preprocessor.imagePath",
|
|
141
|
-
movieFile: ":preprocessor.movieFile",
|
|
142
|
-
},
|
|
143
|
-
},
|
|
144
|
-
output: {
|
|
145
|
-
agent: "copyAgent",
|
|
146
|
-
inputs: {
|
|
147
|
-
imageFile: ":onComplete.imageFile",
|
|
148
|
-
movieFile: ":onComplete.movieFile",
|
|
149
|
-
},
|
|
150
|
-
isResult: true,
|
|
151
|
-
},
|
|
152
|
-
},
|
|
153
|
-
},
|
|
161
|
+
graph: beat_graph_data,
|
|
154
162
|
},
|
|
155
163
|
mergeResult: {
|
|
164
|
+
isResult: true,
|
|
156
165
|
agent: (namedInputs) => {
|
|
157
166
|
const { array, context } = namedInputs;
|
|
158
167
|
const { studio } = context;
|
|
@@ -207,10 +216,8 @@ const googleAuth = async () => {
|
|
|
207
216
|
throw error;
|
|
208
217
|
}
|
|
209
218
|
};
|
|
210
|
-
const
|
|
211
|
-
const { studio
|
|
212
|
-
const { outDirPath, imageDirPath } = fileDirs;
|
|
213
|
-
mkdir(`${imageDirPath}/${studio.filename}`);
|
|
219
|
+
const graphOption = async (context) => {
|
|
220
|
+
const { studio } = context;
|
|
214
221
|
const agentFilters = [
|
|
215
222
|
{
|
|
216
223
|
name: "fileCacheAgentFilter",
|
|
@@ -237,12 +244,13 @@ const generateImages = async (context, callbacks) => {
|
|
|
237
244
|
},
|
|
238
245
|
};
|
|
239
246
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
}
|
|
247
|
+
return options;
|
|
248
|
+
};
|
|
249
|
+
const prepareGenerateImages = async (context) => {
|
|
250
|
+
const { studio, fileDirs } = context;
|
|
251
|
+
const { outDirPath, imageDirPath } = fileDirs;
|
|
252
|
+
mkdir(`${imageDirPath}/${studio.filename}`);
|
|
253
|
+
const imageAgentInfo = MulmoScriptMethods.getImageAgentInfo(studio.script, context.dryRun);
|
|
246
254
|
const imageRefs = {};
|
|
247
255
|
const images = studio.script.imageParams?.images;
|
|
248
256
|
if (images) {
|
|
@@ -285,11 +293,26 @@ const generateImages = async (context, callbacks) => {
|
|
|
285
293
|
const injections = {
|
|
286
294
|
context,
|
|
287
295
|
imageAgentInfo,
|
|
296
|
+
movieAgentInfo: {
|
|
297
|
+
agent: context.dryRun ? "mediaMockAgent" : "movieGoogleAgent",
|
|
298
|
+
},
|
|
288
299
|
outputStudioFilePath: getOutputStudioFilePath(outDirPath, studio.filename),
|
|
289
300
|
imageDirPath,
|
|
290
301
|
imageRefs,
|
|
291
302
|
};
|
|
292
|
-
|
|
303
|
+
return injections;
|
|
304
|
+
};
|
|
305
|
+
const generateImages = async (context, callbacks) => {
|
|
306
|
+
const imageAgentInfo = MulmoScriptMethods.getImageAgentInfo(context.studio.script);
|
|
307
|
+
if (imageAgentInfo.provider === "openai") {
|
|
308
|
+
// NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
|
|
309
|
+
// dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
|
|
310
|
+
// gpt-image-1：3,000,000 TPM、150 images per minute
|
|
311
|
+
graph_data.concurrency = imageAgentInfo.imageParams.model === "dall-e-3" ? 4 : 16;
|
|
312
|
+
}
|
|
313
|
+
const options = await graphOption(context);
|
|
314
|
+
const injections = await prepareGenerateImages(context);
|
|
315
|
+
const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
|
|
293
316
|
Object.keys(injections).forEach((key) => {
|
|
294
317
|
graph.injectValue(key, injections[key]);
|
|
295
318
|
});
|
|
@@ -298,7 +321,8 @@ const generateImages = async (context, callbacks) => {
|
|
|
298
321
|
graph.registerCallback(callback);
|
|
299
322
|
});
|
|
300
323
|
}
|
|
301
|
-
await graph.run();
|
|
324
|
+
const res = await graph.run();
|
|
325
|
+
return res.mergeResult;
|
|
302
326
|
};
|
|
303
327
|
export const images = async (context, callbacks) => {
|
|
304
328
|
try {
|
|
@@ -309,3 +333,21 @@ export const images = async (context, callbacks) => {
|
|
|
309
333
|
MulmoStudioContextMethods.setSessionState(context, "image", false);
|
|
310
334
|
}
|
|
311
335
|
};
|
|
336
|
+
export const generateBeatImage = async (index, context, callbacks) => {
|
|
337
|
+
const options = await graphOption(context);
|
|
338
|
+
const injections = await prepareGenerateImages(context);
|
|
339
|
+
const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
|
|
340
|
+
Object.keys(injections).forEach((key) => {
|
|
341
|
+
if ("outputStudioFilePath" !== key) {
|
|
342
|
+
graph.injectValue(key, injections[key]);
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
graph.injectValue("__mapIndex", index);
|
|
346
|
+
graph.injectValue("beat", context.studio.script.beats[index]);
|
|
347
|
+
if (callbacks) {
|
|
348
|
+
callbacks.forEach((callback) => {
|
|
349
|
+
graph.registerCallback(callback);
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
await graph.run();
|
|
353
|
+
};
|
package/lib/actions/movie.d.ts
CHANGED
|
@@ -7,4 +7,5 @@ export declare const getAudioPart: (inputIndex: number, duration: number, delay:
|
|
|
7
7
|
audioId: string;
|
|
8
8
|
audioPart: string;
|
|
9
9
|
};
|
|
10
|
+
export declare const movieFilePath: (context: MulmoStudioContext) => string;
|
|
10
11
|
export declare const movie: (context: MulmoStudioContext) => Promise<void>;
|