mulmocast 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/assets/html/pdf_handout.html +85 -0
- package/assets/html/pdf_slide.html +55 -0
- package/assets/html/pdf_talk.html +76 -0
- package/lib/actions/audio.js +2 -1
- package/lib/actions/images.d.ts +85 -1
- package/lib/actions/images.js +130 -88
- package/lib/actions/movie.js +9 -2
- package/lib/actions/pdf.d.ts +1 -0
- package/lib/actions/pdf.js +132 -202
- package/lib/agents/combine_audio_files_agent.js +10 -8
- package/lib/agents/image_mock_agent.d.ts +4 -0
- package/lib/agents/image_mock_agent.js +18 -0
- package/lib/agents/index.d.ts +3 -1
- package/lib/agents/index.js +3 -1
- package/lib/agents/media_mock_agent.d.ts +4 -0
- package/lib/agents/media_mock_agent.js +18 -0
- package/lib/agents/tts_openai_agent.js +9 -1
- package/lib/cli/commands/audio/builder.d.ts +2 -0
- package/lib/cli/commands/image/builder.d.ts +2 -0
- package/lib/cli/commands/movie/builder.d.ts +2 -0
- package/lib/cli/commands/pdf/builder.d.ts +2 -0
- package/lib/cli/commands/translate/builder.d.ts +2 -0
- package/lib/cli/common.d.ts +2 -0
- package/lib/cli/common.js +5 -0
- package/lib/cli/helpers.js +1 -0
- package/lib/methods/mulmo_script.d.ts +1 -1
- package/lib/methods/mulmo_script.js +2 -2
- package/lib/types/type.d.ts +1 -0
- package/lib/utils/filters.js +1 -0
- package/lib/utils/markdown.js +1 -1
- package/package.json +7 -8
- package/assets/font/NotoSansJP-Regular.ttf +0 -0
package/README.md
CHANGED
|
@@ -103,9 +103,13 @@ GOOGLE_PROJECT_ID=your_google_project_id
|
|
|
103
103
|
|
|
104
104
|
See also [pre-requisites for Google's image generation model](./docs/pre-requisites-google.md)
|
|
105
105
|
|
|
106
|
-
#### (Optional) For
|
|
106
|
+
#### (Optional) For TTS models
|
|
107
107
|
```bash
|
|
108
|
+
# For Nijivoice TTS
|
|
108
109
|
NIJIVOICE_API_KEY=your_nijivoice_api_key
|
|
110
|
+
|
|
111
|
+
# For ElevenLabs TTS
|
|
112
|
+
ELEVENLABS_API_KEY=your_elevenlabs_api_key
|
|
109
113
|
```
|
|
110
114
|
|
|
111
115
|
#### (Optional) to access web in mulmo tool
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="${lang}">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>${title}</title>
|
|
7
|
+
<style>
|
|
8
|
+
@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+JP:wght@400;700&display=swap');
|
|
9
|
+
|
|
10
|
+
* {
|
|
11
|
+
margin: 0;
|
|
12
|
+
padding: 0;
|
|
13
|
+
box-sizing: border-box;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
body {
|
|
17
|
+
font-family: 'Noto Sans JP', sans-serif;
|
|
18
|
+
font-size: 16px;
|
|
19
|
+
line-height: 1.6;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #fff;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
@page {
|
|
25
|
+
size: ${page_size};
|
|
26
|
+
margin: 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
.page {
|
|
30
|
+
page-break-after: always;
|
|
31
|
+
width: 100%;
|
|
32
|
+
height: 100vh;
|
|
33
|
+
position: relative;
|
|
34
|
+
overflow: hidden;
|
|
35
|
+
padding: 15px;
|
|
36
|
+
display: ${page_layout};
|
|
37
|
+
${page_direction}
|
|
38
|
+
gap: 15px;
|
|
39
|
+
background: #fff;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
.page:last-child {
|
|
43
|
+
page-break-after: avoid;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
img {
|
|
47
|
+
max-width: 100%;
|
|
48
|
+
max-height: 100%;
|
|
49
|
+
object-fit: contain;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
.handout-item {
|
|
53
|
+
display: flex;
|
|
54
|
+
flex-direction: ${flex_direction};
|
|
55
|
+
border: 1px solid #ddd;
|
|
56
|
+
overflow: hidden;
|
|
57
|
+
${item_flex}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.handout-image {
|
|
61
|
+
${image_size}
|
|
62
|
+
display: flex;
|
|
63
|
+
align-items: center;
|
|
64
|
+
justify-content: center;
|
|
65
|
+
background: #f9f9f9;
|
|
66
|
+
padding: 5px;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
.handout-text {
|
|
70
|
+
${text_size}
|
|
71
|
+
padding: 8px;
|
|
72
|
+
font-size: 14px;
|
|
73
|
+
overflow: hidden;
|
|
74
|
+
background: #fff;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
.handout-text p {
|
|
78
|
+
margin: 0.3em 0;
|
|
79
|
+
}
|
|
80
|
+
</style>
|
|
81
|
+
</head>
|
|
82
|
+
<body>
|
|
83
|
+
${pages}
|
|
84
|
+
</body>
|
|
85
|
+
</html>
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="${lang}">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>${title}</title>
|
|
7
|
+
<style>
|
|
8
|
+
@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+JP:wght@400;700&display=swap');
|
|
9
|
+
|
|
10
|
+
* {
|
|
11
|
+
margin: 0;
|
|
12
|
+
padding: 0;
|
|
13
|
+
box-sizing: border-box;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
body {
|
|
17
|
+
font-family: 'Noto Sans JP', sans-serif;
|
|
18
|
+
font-size: 14px;
|
|
19
|
+
line-height: 1.6;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #fff;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
@page {
|
|
25
|
+
size: ${page_size};
|
|
26
|
+
margin: 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
.page {
|
|
30
|
+
page-break-after: always;
|
|
31
|
+
width: 100%;
|
|
32
|
+
height: 100vh;
|
|
33
|
+
position: relative;
|
|
34
|
+
overflow: hidden;
|
|
35
|
+
display: flex;
|
|
36
|
+
align-items: center;
|
|
37
|
+
justify-content: center;
|
|
38
|
+
background: #fff;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
.page:last-child {
|
|
42
|
+
page-break-after: avoid;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
img {
|
|
46
|
+
max-width: 100%;
|
|
47
|
+
max-height: 100%;
|
|
48
|
+
object-fit: contain;
|
|
49
|
+
}
|
|
50
|
+
</style>
|
|
51
|
+
</head>
|
|
52
|
+
<body>
|
|
53
|
+
${pages}
|
|
54
|
+
</body>
|
|
55
|
+
</html>
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="${lang}">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>${title}</title>
|
|
7
|
+
<style>
|
|
8
|
+
@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+JP:wght@400;700&display=swap');
|
|
9
|
+
|
|
10
|
+
* {
|
|
11
|
+
margin: 0;
|
|
12
|
+
padding: 0;
|
|
13
|
+
box-sizing: border-box;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
body {
|
|
17
|
+
font-family: 'Noto Sans JP', sans-serif;
|
|
18
|
+
font-size: 17px;
|
|
19
|
+
line-height: 1.4;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #fff;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
@page {
|
|
25
|
+
size: ${page_size};
|
|
26
|
+
margin: 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
.page {
|
|
30
|
+
page-break-after: always;
|
|
31
|
+
width: 100%;
|
|
32
|
+
height: 100vh;
|
|
33
|
+
position: relative;
|
|
34
|
+
overflow: hidden;
|
|
35
|
+
padding: 20px;
|
|
36
|
+
display: flex;
|
|
37
|
+
flex-direction: column;
|
|
38
|
+
background: #fff;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
.page:last-child {
|
|
42
|
+
page-break-after: avoid;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
img {
|
|
46
|
+
max-width: 100%;
|
|
47
|
+
max-height: 100%;
|
|
48
|
+
object-fit: contain;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
.image-container {
|
|
52
|
+
flex: 1;
|
|
53
|
+
display: flex;
|
|
54
|
+
align-items: center;
|
|
55
|
+
justify-content: center;
|
|
56
|
+
margin-bottom: 20px;
|
|
57
|
+
border: 1px solid #ddd;
|
|
58
|
+
background: #f9f9f9;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
.text-container {
|
|
62
|
+
padding: 10px;
|
|
63
|
+
background: #fff;
|
|
64
|
+
border-top: 2px solid #333;
|
|
65
|
+
min-height: 120px;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
.text-container p {
|
|
69
|
+
margin: 0.5em 0;
|
|
70
|
+
}
|
|
71
|
+
</style>
|
|
72
|
+
</head>
|
|
73
|
+
<body>
|
|
74
|
+
${pages}
|
|
75
|
+
</body>
|
|
76
|
+
</html>
|
package/lib/actions/audio.js
CHANGED
|
@@ -22,6 +22,7 @@ const provider_to_agent = {
|
|
|
22
22
|
openai: "ttsOpenaiAgent",
|
|
23
23
|
google: "ttsGoogleAgent",
|
|
24
24
|
elevenlabs: "ttsElevenlabsAgent",
|
|
25
|
+
mock: "mediaMockAgent",
|
|
25
26
|
};
|
|
26
27
|
const getAudioPath = (context, beat, audioFile, audioDirPath) => {
|
|
27
28
|
if (beat.audio?.type === "audio") {
|
|
@@ -31,7 +32,7 @@ const getAudioPath = (context, beat, audioFile, audioDirPath) => {
|
|
|
31
32
|
}
|
|
32
33
|
throw new Error("Invalid audio source");
|
|
33
34
|
}
|
|
34
|
-
if (beat.text === "") {
|
|
35
|
+
if (beat.text === undefined || beat.text === "") {
|
|
35
36
|
return undefined; // It indicates that the audio is not needed.
|
|
36
37
|
}
|
|
37
38
|
return getAudioSegmentFilePath(audioDirPath, context.studio.filename, audioFile);
|
package/lib/actions/images.d.ts
CHANGED
|
@@ -1,3 +1,87 @@
|
|
|
1
1
|
import type { CallbackFunction } from "graphai";
|
|
2
|
-
import { MulmoStudioContext } from "../types/index.js";
|
|
2
|
+
import { MulmoStudioContext, MulmoBeat, Text2ImageAgentInfo } from "../types/index.js";
|
|
3
|
+
export declare const imagePreprocessAgent: (namedInputs: {
|
|
4
|
+
context: MulmoStudioContext;
|
|
5
|
+
beat: MulmoBeat;
|
|
6
|
+
index: number;
|
|
7
|
+
suffix: string;
|
|
8
|
+
imageDirPath: string;
|
|
9
|
+
imageAgentInfo: Text2ImageAgentInfo;
|
|
10
|
+
imageRefs: Record<string, string>;
|
|
11
|
+
}) => Promise<{
|
|
12
|
+
imageParams: {
|
|
13
|
+
model?: string | undefined;
|
|
14
|
+
style?: string | undefined;
|
|
15
|
+
moderation?: string | undefined;
|
|
16
|
+
images?: Record<string, {
|
|
17
|
+
type: "image";
|
|
18
|
+
source: {
|
|
19
|
+
url: string;
|
|
20
|
+
kind: "url";
|
|
21
|
+
} | {
|
|
22
|
+
kind: "base64";
|
|
23
|
+
data: string;
|
|
24
|
+
} | {
|
|
25
|
+
text: string;
|
|
26
|
+
kind: "text";
|
|
27
|
+
} | {
|
|
28
|
+
path: string;
|
|
29
|
+
kind: "path";
|
|
30
|
+
};
|
|
31
|
+
}> | undefined;
|
|
32
|
+
};
|
|
33
|
+
movieFile: string | undefined;
|
|
34
|
+
imagePath: string | undefined;
|
|
35
|
+
} | {
|
|
36
|
+
images: string[];
|
|
37
|
+
imageParams: {
|
|
38
|
+
model?: string | undefined;
|
|
39
|
+
style?: string | undefined;
|
|
40
|
+
moderation?: string | undefined;
|
|
41
|
+
images?: Record<string, {
|
|
42
|
+
type: "image";
|
|
43
|
+
source: {
|
|
44
|
+
url: string;
|
|
45
|
+
kind: "url";
|
|
46
|
+
} | {
|
|
47
|
+
kind: "base64";
|
|
48
|
+
data: string;
|
|
49
|
+
} | {
|
|
50
|
+
text: string;
|
|
51
|
+
kind: "text";
|
|
52
|
+
} | {
|
|
53
|
+
path: string;
|
|
54
|
+
kind: "path";
|
|
55
|
+
};
|
|
56
|
+
}> | undefined;
|
|
57
|
+
};
|
|
58
|
+
movieFile: string | undefined;
|
|
59
|
+
} | {
|
|
60
|
+
images: string[];
|
|
61
|
+
imageParams: {
|
|
62
|
+
model?: string | undefined;
|
|
63
|
+
style?: string | undefined;
|
|
64
|
+
moderation?: string | undefined;
|
|
65
|
+
images?: Record<string, {
|
|
66
|
+
type: "image";
|
|
67
|
+
source: {
|
|
68
|
+
url: string;
|
|
69
|
+
kind: "url";
|
|
70
|
+
} | {
|
|
71
|
+
kind: "base64";
|
|
72
|
+
data: string;
|
|
73
|
+
} | {
|
|
74
|
+
text: string;
|
|
75
|
+
kind: "text";
|
|
76
|
+
} | {
|
|
77
|
+
path: string;
|
|
78
|
+
kind: "path";
|
|
79
|
+
};
|
|
80
|
+
}> | undefined;
|
|
81
|
+
};
|
|
82
|
+
movieFile: string | undefined;
|
|
83
|
+
imagePath: string;
|
|
84
|
+
prompt: string;
|
|
85
|
+
}>;
|
|
3
86
|
export declare const images: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
|
|
87
|
+
export declare const generateBeatImage: (index: number, context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
|
package/lib/actions/images.js
CHANGED
|
@@ -5,9 +5,7 @@ import * as agents from "@graphai/vanilla";
|
|
|
5
5
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
6
6
|
import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
|
|
7
7
|
import { fileCacheAgentFilter } from "../utils/filters.js";
|
|
8
|
-
import imageGoogleAgent from "../agents/
|
|
9
|
-
import imageOpenaiAgent from "../agents/image_openai_agent.js";
|
|
10
|
-
import movieGoogleAgent from "../agents/movie_google_agent.js";
|
|
8
|
+
import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent } from "../agents/index.js";
|
|
11
9
|
import { MulmoScriptMethods, MulmoStudioContextMethods } from "../methods/index.js";
|
|
12
10
|
import { imagePlugins } from "../utils/image_plugins/index.js";
|
|
13
11
|
import { imagePrompt } from "../utils/prompt.js";
|
|
@@ -21,7 +19,7 @@ const htmlStyle = (script, beat) => {
|
|
|
21
19
|
textSlideStyle: MulmoScriptMethods.getTextSlideStyle(script, beat),
|
|
22
20
|
};
|
|
23
21
|
};
|
|
24
|
-
const imagePreprocessAgent = async (namedInputs) => {
|
|
22
|
+
export const imagePreprocessAgent = async (namedInputs) => {
|
|
25
23
|
const { context, beat, index, suffix, imageDirPath, imageAgentInfo, imageRefs } = namedInputs;
|
|
26
24
|
const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
|
|
27
25
|
const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
|
|
@@ -56,6 +54,85 @@ const imagePreprocessAgent = async (namedInputs) => {
|
|
|
56
54
|
const prompt = imagePrompt(beat, imageParams.style);
|
|
57
55
|
return { imagePath, prompt, ...returnValue, images };
|
|
58
56
|
};
|
|
57
|
+
const beat_graph_data = {
|
|
58
|
+
version: 0.5,
|
|
59
|
+
concurrency: 4,
|
|
60
|
+
nodes: {
|
|
61
|
+
context: {},
|
|
62
|
+
imageDirPath: {},
|
|
63
|
+
imageAgentInfo: {},
|
|
64
|
+
movieAgentInfo: {},
|
|
65
|
+
imageRefs: {},
|
|
66
|
+
beat: {},
|
|
67
|
+
__mapIndex: {},
|
|
68
|
+
preprocessor: {
|
|
69
|
+
agent: imagePreprocessAgent,
|
|
70
|
+
inputs: {
|
|
71
|
+
context: ":context",
|
|
72
|
+
beat: ":beat",
|
|
73
|
+
index: ":__mapIndex",
|
|
74
|
+
suffix: "p",
|
|
75
|
+
imageDirPath: ":imageDirPath",
|
|
76
|
+
imageAgentInfo: ":imageAgentInfo",
|
|
77
|
+
imageRefs: ":imageRefs",
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
imageGenerator: {
|
|
81
|
+
if: ":preprocessor.prompt",
|
|
82
|
+
agent: ":imageAgentInfo.agent",
|
|
83
|
+
retry: 3,
|
|
84
|
+
inputs: {
|
|
85
|
+
prompt: ":preprocessor.prompt",
|
|
86
|
+
images: ":preprocessor.images",
|
|
87
|
+
file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
|
|
88
|
+
text: ":preprocessor.prompt", // only for fileCacheAgentFilter
|
|
89
|
+
force: ":context.force", // only for fileCacheAgentFilter
|
|
90
|
+
mulmoContext: ":context", // for fileCacheAgentFilter
|
|
91
|
+
index: ":__mapIndex", // for fileCacheAgentFilter
|
|
92
|
+
sessionType: "image", // for fileCacheAgentFilter
|
|
93
|
+
params: {
|
|
94
|
+
model: ":preprocessor.imageParams.model",
|
|
95
|
+
moderation: ":preprocessor.imageParams.moderation",
|
|
96
|
+
canvasSize: ":context.studio.script.canvasSize",
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
defaultValue: {},
|
|
100
|
+
},
|
|
101
|
+
movieGenerator: {
|
|
102
|
+
if: ":preprocessor.movieFile",
|
|
103
|
+
agent: ":movieAgentInfo.agent",
|
|
104
|
+
inputs: {
|
|
105
|
+
onComplete: ":imageGenerator", // to wait for imageGenerator to finish
|
|
106
|
+
prompt: ":beat.moviePrompt",
|
|
107
|
+
imagePath: ":preprocessor.imagePath",
|
|
108
|
+
file: ":preprocessor.movieFile",
|
|
109
|
+
studio: ":context.studio", // for cache
|
|
110
|
+
mulmoContext: ":context", // for fileCacheAgentFilter
|
|
111
|
+
index: ":__mapIndex", // for cache
|
|
112
|
+
sessionType: "movie", // for cache
|
|
113
|
+
params: {
|
|
114
|
+
model: ":context.studio.script.movieParams.model",
|
|
115
|
+
duration: ":beat.duration",
|
|
116
|
+
canvasSize: ":context.studio.script.canvasSize",
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
defaultValue: {},
|
|
120
|
+
},
|
|
121
|
+
output: {
|
|
122
|
+
agent: "copyAgent",
|
|
123
|
+
inputs: {
|
|
124
|
+
onComplete: ":movieGenerator", // to wait for movieGenerator to finish
|
|
125
|
+
imageFile: ":preprocessor.imagePath",
|
|
126
|
+
movieFile: ":preprocessor.movieFile",
|
|
127
|
+
},
|
|
128
|
+
output: {
|
|
129
|
+
imageFile: ".imageFile",
|
|
130
|
+
movieFile: ".movieFile",
|
|
131
|
+
},
|
|
132
|
+
isResult: true,
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
};
|
|
59
136
|
const graph_data = {
|
|
60
137
|
version: 0.5,
|
|
61
138
|
concurrency: 4,
|
|
@@ -63,6 +140,7 @@ const graph_data = {
|
|
|
63
140
|
context: {},
|
|
64
141
|
imageDirPath: {},
|
|
65
142
|
imageAgentInfo: {},
|
|
143
|
+
movieAgentInfo: {},
|
|
66
144
|
outputStudioFilePath: {},
|
|
67
145
|
imageRefs: {},
|
|
68
146
|
map: {
|
|
@@ -71,6 +149,7 @@ const graph_data = {
|
|
|
71
149
|
rows: ":context.studio.script.beats",
|
|
72
150
|
context: ":context",
|
|
73
151
|
imageAgentInfo: ":imageAgentInfo",
|
|
152
|
+
movieAgentInfo: ":movieAgentInfo",
|
|
74
153
|
imageDirPath: ":imageDirPath",
|
|
75
154
|
imageRefs: ":imageRefs",
|
|
76
155
|
},
|
|
@@ -79,80 +158,10 @@ const graph_data = {
|
|
|
79
158
|
rowKey: "beat",
|
|
80
159
|
compositeResult: true,
|
|
81
160
|
},
|
|
82
|
-
graph:
|
|
83
|
-
nodes: {
|
|
84
|
-
preprocessor: {
|
|
85
|
-
agent: imagePreprocessAgent,
|
|
86
|
-
inputs: {
|
|
87
|
-
context: ":context",
|
|
88
|
-
beat: ":beat",
|
|
89
|
-
index: ":__mapIndex",
|
|
90
|
-
suffix: "p",
|
|
91
|
-
imageDirPath: ":imageDirPath",
|
|
92
|
-
imageAgentInfo: ":imageAgentInfo",
|
|
93
|
-
imageRefs: ":imageRefs",
|
|
94
|
-
},
|
|
95
|
-
},
|
|
96
|
-
imageGenerator: {
|
|
97
|
-
if: ":preprocessor.prompt",
|
|
98
|
-
agent: ":imageAgentInfo.agent",
|
|
99
|
-
retry: 3,
|
|
100
|
-
inputs: {
|
|
101
|
-
prompt: ":preprocessor.prompt",
|
|
102
|
-
images: ":preprocessor.images",
|
|
103
|
-
file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
|
|
104
|
-
text: ":preprocessor.prompt", // only for fileCacheAgentFilter
|
|
105
|
-
force: ":context.force", // only for fileCacheAgentFilter
|
|
106
|
-
mulmoContext: ":context", // for fileCacheAgentFilter
|
|
107
|
-
index: ":__mapIndex", // for fileCacheAgentFilter
|
|
108
|
-
sessionType: "image", // for fileCacheAgentFilter
|
|
109
|
-
params: {
|
|
110
|
-
model: ":preprocessor.imageParams.model",
|
|
111
|
-
moderation: ":preprocessor.imageParams.moderation",
|
|
112
|
-
canvasSize: ":context.studio.script.canvasSize",
|
|
113
|
-
},
|
|
114
|
-
},
|
|
115
|
-
defaultValue: {},
|
|
116
|
-
},
|
|
117
|
-
movieGenerator: {
|
|
118
|
-
if: ":preprocessor.movieFile",
|
|
119
|
-
agent: "movieGoogleAgent",
|
|
120
|
-
inputs: {
|
|
121
|
-
onComplete: ":imageGenerator", // to wait for imageGenerator to finish
|
|
122
|
-
prompt: ":beat.moviePrompt",
|
|
123
|
-
imagePath: ":preprocessor.imagePath",
|
|
124
|
-
file: ":preprocessor.movieFile",
|
|
125
|
-
studio: ":context.studio", // for cache
|
|
126
|
-
index: ":__mapIndex", // for cache
|
|
127
|
-
sessionType: "movie", // for cache
|
|
128
|
-
params: {
|
|
129
|
-
model: ":context.studio.script.movieParams.model",
|
|
130
|
-
duration: ":beat.duration",
|
|
131
|
-
canvasSize: ":context.studio.script.canvasSize",
|
|
132
|
-
},
|
|
133
|
-
},
|
|
134
|
-
defaultValue: {},
|
|
135
|
-
},
|
|
136
|
-
onComplete: {
|
|
137
|
-
agent: "copyAgent",
|
|
138
|
-
inputs: {
|
|
139
|
-
onComplete: ":movieGenerator", // to wait for movieGenerator to finish
|
|
140
|
-
imageFile: ":preprocessor.imagePath",
|
|
141
|
-
movieFile: ":preprocessor.movieFile",
|
|
142
|
-
},
|
|
143
|
-
},
|
|
144
|
-
output: {
|
|
145
|
-
agent: "copyAgent",
|
|
146
|
-
inputs: {
|
|
147
|
-
imageFile: ":onComplete.imageFile",
|
|
148
|
-
movieFile: ":onComplete.movieFile",
|
|
149
|
-
},
|
|
150
|
-
isResult: true,
|
|
151
|
-
},
|
|
152
|
-
},
|
|
153
|
-
},
|
|
161
|
+
graph: beat_graph_data,
|
|
154
162
|
},
|
|
155
163
|
mergeResult: {
|
|
164
|
+
isResult: true,
|
|
156
165
|
agent: (namedInputs) => {
|
|
157
166
|
const { array, context } = namedInputs;
|
|
158
167
|
const { studio } = context;
|
|
@@ -207,10 +216,8 @@ const googleAuth = async () => {
|
|
|
207
216
|
throw error;
|
|
208
217
|
}
|
|
209
218
|
};
|
|
210
|
-
const
|
|
211
|
-
const { studio
|
|
212
|
-
const { outDirPath, imageDirPath } = fileDirs;
|
|
213
|
-
mkdir(`${imageDirPath}/${studio.filename}`);
|
|
219
|
+
const graphOption = async (context) => {
|
|
220
|
+
const { studio } = context;
|
|
214
221
|
const agentFilters = [
|
|
215
222
|
{
|
|
216
223
|
name: "fileCacheAgentFilter",
|
|
@@ -237,12 +244,13 @@ const generateImages = async (context, callbacks) => {
|
|
|
237
244
|
},
|
|
238
245
|
};
|
|
239
246
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
}
|
|
247
|
+
return options;
|
|
248
|
+
};
|
|
249
|
+
const prepareGenerateImages = async (context) => {
|
|
250
|
+
const { studio, fileDirs } = context;
|
|
251
|
+
const { outDirPath, imageDirPath } = fileDirs;
|
|
252
|
+
mkdir(`${imageDirPath}/${studio.filename}`);
|
|
253
|
+
const imageAgentInfo = MulmoScriptMethods.getImageAgentInfo(studio.script, context.dryRun);
|
|
246
254
|
const imageRefs = {};
|
|
247
255
|
const images = studio.script.imageParams?.images;
|
|
248
256
|
if (images) {
|
|
@@ -285,11 +293,26 @@ const generateImages = async (context, callbacks) => {
|
|
|
285
293
|
const injections = {
|
|
286
294
|
context,
|
|
287
295
|
imageAgentInfo,
|
|
296
|
+
movieAgentInfo: {
|
|
297
|
+
agent: context.dryRun ? "mediaMockAgent" : "movieGoogleAgent",
|
|
298
|
+
},
|
|
288
299
|
outputStudioFilePath: getOutputStudioFilePath(outDirPath, studio.filename),
|
|
289
300
|
imageDirPath,
|
|
290
301
|
imageRefs,
|
|
291
302
|
};
|
|
292
|
-
|
|
303
|
+
return injections;
|
|
304
|
+
};
|
|
305
|
+
const generateImages = async (context, callbacks) => {
|
|
306
|
+
const imageAgentInfo = MulmoScriptMethods.getImageAgentInfo(context.studio.script);
|
|
307
|
+
if (imageAgentInfo.provider === "openai") {
|
|
308
|
+
// NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
|
|
309
|
+
// dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
|
|
310
|
+
// gpt-image-1：3,000,000 TPM、150 images per minute
|
|
311
|
+
graph_data.concurrency = imageAgentInfo.imageParams.model === "dall-e-3" ? 4 : 16;
|
|
312
|
+
}
|
|
313
|
+
const options = await graphOption(context);
|
|
314
|
+
const injections = await prepareGenerateImages(context);
|
|
315
|
+
const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
|
|
293
316
|
Object.keys(injections).forEach((key) => {
|
|
294
317
|
graph.injectValue(key, injections[key]);
|
|
295
318
|
});
|
|
@@ -298,7 +321,8 @@ const generateImages = async (context, callbacks) => {
|
|
|
298
321
|
graph.registerCallback(callback);
|
|
299
322
|
});
|
|
300
323
|
}
|
|
301
|
-
await graph.run();
|
|
324
|
+
const res = await graph.run();
|
|
325
|
+
return res.mergeResult;
|
|
302
326
|
};
|
|
303
327
|
export const images = async (context, callbacks) => {
|
|
304
328
|
try {
|
|
@@ -309,3 +333,21 @@ export const images = async (context, callbacks) => {
|
|
|
309
333
|
MulmoStudioContextMethods.setSessionState(context, "image", false);
|
|
310
334
|
}
|
|
311
335
|
};
|
|
336
|
+
export const generateBeatImage = async (index, context, callbacks) => {
|
|
337
|
+
const options = await graphOption(context);
|
|
338
|
+
const injections = await prepareGenerateImages(context);
|
|
339
|
+
const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
|
|
340
|
+
Object.keys(injections).forEach((key) => {
|
|
341
|
+
if ("outputStudioFilePath" !== key) {
|
|
342
|
+
graph.injectValue(key, injections[key]);
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
graph.injectValue("__mapIndex", index);
|
|
346
|
+
graph.injectValue("beat", context.studio.script.beats[index]);
|
|
347
|
+
if (callbacks) {
|
|
348
|
+
callbacks.forEach((callback) => {
|
|
349
|
+
graph.registerCallback(callback);
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
await graph.run();
|
|
353
|
+
};
|
package/lib/actions/movie.js
CHANGED
|
@@ -110,7 +110,14 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
110
110
|
const sourceId = filterComplexVideoIds.pop();
|
|
111
111
|
ffmpegContext.filterComplex.push(`[${sourceId}]split=2[${sourceId}_0][${sourceId}_1]`);
|
|
112
112
|
filterComplexVideoIds.push(`${sourceId}_0`);
|
|
113
|
-
|
|
113
|
+
if (mediaType === "movie") {
|
|
114
|
+
// For movie beats, extract the last frame for transition
|
|
115
|
+
ffmpegContext.filterComplex.push(`[${sourceId}_1]reverse,select='eq(n,0)',reverse,tpad=stop_mode=clone:stop_duration=${duration},fps=30,setpts=PTS-STARTPTS[${sourceId}_2]`);
|
|
116
|
+
transitionVideoIds.push(`${sourceId}_2`);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
transitionVideoIds.push(`${sourceId}_1`);
|
|
120
|
+
}
|
|
114
121
|
}
|
|
115
122
|
if (beat.image?.type == "movie" && beat.image.mixAudio > 0.0) {
|
|
116
123
|
const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, beat.image.mixAudio);
|
|
@@ -133,7 +140,6 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
133
140
|
return transitionVideoIds.reduce((acc, transitionVideoId, index) => {
|
|
134
141
|
const transitionStartTime = beatTimestamps[index + 1] - 0.05; // 0.05 is to avoid flickering
|
|
135
142
|
const processedVideoId = `${transitionVideoId}_f`;
|
|
136
|
-
// TODO: This mechanism does not work for video beats yet. It works only with image beats.
|
|
137
143
|
// If we can to add other transition types than fade, we need to add them here.
|
|
138
144
|
ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${transition.duration}:alpha=1,setpts=PTS-STARTPTS+${transitionStartTime}/TB[${processedVideoId}]`);
|
|
139
145
|
const outputId = `${transitionVideoId}_o`;
|
|
@@ -156,6 +162,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
156
162
|
}
|
|
157
163
|
return artifactAudioId;
|
|
158
164
|
})();
|
|
165
|
+
// GraphAILogger.debug("filterComplex", ffmpegContext.filterComplex);
|
|
159
166
|
await FfmpegContextGenerateOutput(ffmpegContext, outputVideoPath, getOutputOption(ffmpegContextAudioId, mixedVideoId));
|
|
160
167
|
const end = performance.now();
|
|
161
168
|
GraphAILogger.info(`Video created successfully! ${Math.round(end - start) / 1000} sec`);
|
package/lib/actions/pdf.d.ts
CHANGED
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
import { MulmoStudioContext, PDFMode, PDFSize } from "../types/index.js";
|
|
2
|
+
export declare const pdfFilePath: (context: MulmoStudioContext, pdfMode: PDFMode) => string;
|
|
2
3
|
export declare const pdf: (context: MulmoStudioContext, pdfMode: PDFMode, pdfSize: PDFSize) => Promise<void>;
|