mulmocast 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -9
- package/assets/font/NotoSansJP-Regular.ttf +0 -0
- package/assets/html/chart.html +1 -10
- package/assets/html/mermaid.html +1 -13
- package/assets/templates/business.json +16 -27
- package/assets/templates/coding.json +58 -21
- package/lib/actions/audio.d.ts +1 -1
- package/lib/actions/audio.js +43 -27
- package/lib/actions/images.js +20 -26
- package/lib/actions/index.d.ts +5 -0
- package/lib/actions/index.js +5 -0
- package/lib/actions/movie.d.ts +9 -1
- package/lib/actions/movie.js +97 -38
- package/lib/actions/pdf.d.ts +2 -0
- package/lib/actions/pdf.js +211 -0
- package/lib/actions/translate.js +22 -9
- package/lib/agents/combine_audio_files_agent.js +13 -22
- package/lib/cli/args.d.ts +3 -1
- package/lib/cli/args.js +49 -34
- package/lib/cli/cli.d.ts +15 -0
- package/lib/cli/cli.js +44 -47
- package/lib/cli/run.d.ts +1 -0
- package/lib/cli/run.js +2 -0
- package/lib/cli/tool-args.d.ts +2 -0
- package/lib/cli/tool-args.js +12 -2
- package/lib/cli/tool-cli.js +6 -4
- package/lib/methods/index.d.ts +1 -0
- package/lib/methods/index.js +1 -0
- package/lib/methods/mulmo_media_source.d.ts +4 -0
- package/lib/methods/mulmo_media_source.js +21 -0
- package/lib/methods/mulmo_script.d.ts +2 -6
- package/lib/methods/mulmo_script.js +12 -5
- package/lib/tools/create_mulmo_script_interactively.d.ts +1 -1
- package/lib/tools/create_mulmo_script_interactively.js +61 -20
- package/lib/types/index.d.ts +1 -0
- package/lib/types/index.js +1 -0
- package/lib/types/schema.d.ts +3626 -3162
- package/lib/types/schema.js +75 -41
- package/lib/types/type.d.ts +28 -1
- package/lib/utils/const.d.ts +2 -0
- package/lib/utils/const.js +2 -0
- package/lib/utils/file.d.ts +4 -1
- package/lib/utils/file.js +15 -1
- package/lib/utils/filters.js +1 -1
- package/lib/utils/image_plugins/chart.d.ts +3 -0
- package/lib/utils/image_plugins/chart.js +18 -0
- package/lib/utils/image_plugins/image.d.ts +2 -0
- package/lib/utils/image_plugins/image.js +3 -0
- package/lib/utils/image_plugins/index.d.ts +7 -0
- package/lib/utils/image_plugins/index.js +7 -0
- package/lib/utils/image_plugins/markdown.d.ts +3 -0
- package/lib/utils/image_plugins/markdown.js +11 -0
- package/lib/utils/image_plugins/mermaid.d.ts +3 -0
- package/lib/utils/image_plugins/mermaid.js +21 -0
- package/lib/utils/image_plugins/movie.d.ts +2 -0
- package/lib/utils/image_plugins/movie.js +3 -0
- package/lib/utils/image_plugins/source.d.ts +4 -0
- package/lib/utils/image_plugins/source.js +15 -0
- package/lib/utils/image_plugins/text_slide.d.ts +3 -0
- package/lib/utils/image_plugins/text_slide.js +12 -0
- package/lib/utils/image_plugins/type_guards.d.ts +6 -0
- package/lib/utils/image_plugins/type_guards.js +21 -0
- package/lib/utils/markdown.js +4 -1
- package/lib/utils/pdf.d.ts +8 -0
- package/lib/utils/pdf.js +75 -0
- package/lib/utils/preprocess.d.ts +58 -128
- package/lib/utils/preprocess.js +37 -37
- package/lib/utils/utils.d.ts +12 -0
- package/lib/utils/utils.js +34 -0
- package/package.json +21 -12
- package/lib/tools/seed.d.ts +0 -3
- package/lib/tools/seed.js +0 -201
- package/lib/tools/seed_from_url.d.ts +0 -3
- package/lib/tools/seed_from_url.js +0 -178
- package/lib/tools/seed_from_url2.d.ts +0 -3
- package/lib/tools/seed_from_url2.js +0 -154
- package/lib/utils/image_preprocess.d.ts +0 -14
- package/lib/utils/image_preprocess.js +0 -52
- package/lib/utils/text_hash.d.ts +0 -1
- package/lib/utils/text_hash.js +0 -4
package/lib/actions/movie.js
CHANGED
|
@@ -2,56 +2,115 @@ import ffmpeg from "fluent-ffmpeg";
|
|
|
2
2
|
import { GraphAILogger } from "graphai";
|
|
3
3
|
import { MulmoScriptMethods } from "../methods/index.js";
|
|
4
4
|
import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage } from "../utils/file.js";
|
|
5
|
+
const isMac = process.platform === "darwin";
|
|
6
|
+
const videoCodec = isMac ? "h264_videotoolbox" : "libx264";
|
|
7
|
+
export const getVideoPart = (inputIndex, mediaType, duration, canvasInfo) => {
|
|
8
|
+
const videoId = `v${inputIndex}`;
|
|
9
|
+
return {
|
|
10
|
+
videoId,
|
|
11
|
+
videoPart: `[${inputIndex}:v]` +
|
|
12
|
+
[
|
|
13
|
+
mediaType === "image" ? "loop=loop=-1:size=1:start=0" : "",
|
|
14
|
+
`trim=duration=${duration}`,
|
|
15
|
+
"fps=30",
|
|
16
|
+
"setpts=PTS-STARTPTS",
|
|
17
|
+
`scale=${canvasInfo.width}:${canvasInfo.height}`,
|
|
18
|
+
"setsar=1",
|
|
19
|
+
"format=yuv420p",
|
|
20
|
+
]
|
|
21
|
+
.filter((a) => a)
|
|
22
|
+
.join(",") +
|
|
23
|
+
`[${videoId}]`,
|
|
24
|
+
};
|
|
25
|
+
};
|
|
26
|
+
export const getAudioPart = (inputIndex, duration, delay) => {
|
|
27
|
+
const audioId = `a${inputIndex}`;
|
|
28
|
+
return {
|
|
29
|
+
audioId,
|
|
30
|
+
audioPart: `[${inputIndex}:a]` +
|
|
31
|
+
`atrim=duration=${duration},` + // Trim to beat duration
|
|
32
|
+
`adelay=${delay}|${delay},` +
|
|
33
|
+
`aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo` +
|
|
34
|
+
`[${audioId}]`,
|
|
35
|
+
};
|
|
36
|
+
};
|
|
37
|
+
const getOutputOption = (audioId) => {
|
|
38
|
+
return [
|
|
39
|
+
"-preset veryfast", // Faster encoding
|
|
40
|
+
"-map [v]", // Map the video stream
|
|
41
|
+
`-map ${audioId}`, // Map the audio stream
|
|
42
|
+
`-c:v ${videoCodec}`, // Set video codec
|
|
43
|
+
"-threads 8",
|
|
44
|
+
"-filter_threads 8",
|
|
45
|
+
"-b:v 5M", // bitrate (only for videotoolbox)
|
|
46
|
+
"-bufsize",
|
|
47
|
+
"10M", // Add buffer size for better quality
|
|
48
|
+
"-maxrate",
|
|
49
|
+
"7M", // Maximum bitrate
|
|
50
|
+
"-r 30", // Set frame rate
|
|
51
|
+
"-pix_fmt yuv420p", // Set pixel format for better compatibility
|
|
52
|
+
];
|
|
53
|
+
};
|
|
5
54
|
const createVideo = (audioArtifactFilePath, outputVideoPath, studio) => {
|
|
6
55
|
return new Promise((resolve, reject) => {
|
|
7
56
|
const start = performance.now();
|
|
8
|
-
|
|
57
|
+
const ffmpegContext = {
|
|
58
|
+
command: ffmpeg(),
|
|
59
|
+
inputCount: 0,
|
|
60
|
+
};
|
|
61
|
+
function addInput(input) {
|
|
62
|
+
ffmpegContext.command = ffmpegContext.command.input(input);
|
|
63
|
+
ffmpegContext.inputCount++;
|
|
64
|
+
return ffmpegContext.inputCount - 1; // returned the index of the input
|
|
65
|
+
}
|
|
9
66
|
if (studio.beats.some((beat) => !beat.imageFile)) {
|
|
10
67
|
GraphAILogger.info("beat.imageFile is not set. Please run `yarn run images ${file}` ");
|
|
11
68
|
return;
|
|
12
69
|
}
|
|
13
|
-
// Add each image input
|
|
14
|
-
studio.beats.forEach((beat) => {
|
|
15
|
-
command = command.input(beat.imageFile); // HACK
|
|
16
|
-
});
|
|
17
|
-
const imageCount = studio.beats.length;
|
|
18
70
|
const canvasInfo = MulmoScriptMethods.getCanvasSize(studio.script);
|
|
71
|
+
const padding = MulmoScriptMethods.getPadding(studio.script) / 1000;
|
|
72
|
+
// Add each image input
|
|
19
73
|
const filterComplexParts = [];
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
`
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
74
|
+
const filterComplexVideoIds = [];
|
|
75
|
+
const filterComplexAudioIds = [];
|
|
76
|
+
studio.beats.reduce((timestamp, beat, index) => {
|
|
77
|
+
if (!beat.imageFile || !beat.duration) {
|
|
78
|
+
throw new Error(`beat.imageFile is not set: index=${index}`);
|
|
79
|
+
}
|
|
80
|
+
const inputIndex = addInput(beat.imageFile);
|
|
81
|
+
const mediaType = MulmoScriptMethods.getImageType(studio.script, studio.script.beats[index]);
|
|
82
|
+
const headOrTail = index === 0 || index === studio.beats.length - 1;
|
|
83
|
+
const duration = beat.duration + (headOrTail ? padding : 0);
|
|
84
|
+
const { videoId, videoPart } = getVideoPart(inputIndex, mediaType, duration, canvasInfo);
|
|
85
|
+
filterComplexVideoIds.push(videoId);
|
|
86
|
+
filterComplexParts.push(videoPart);
|
|
87
|
+
if (mediaType === "movie") {
|
|
88
|
+
const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp * 1000);
|
|
89
|
+
filterComplexAudioIds.push(audioId);
|
|
90
|
+
filterComplexParts.push(audioPart);
|
|
91
|
+
}
|
|
92
|
+
return timestamp + duration;
|
|
93
|
+
}, 0);
|
|
94
|
+
// console.log("*** images", images.audioIds);
|
|
33
95
|
// Concatenate the trimmed images
|
|
34
|
-
|
|
35
|
-
|
|
96
|
+
filterComplexParts.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${studio.beats.length}:v=1:a=0[v]`);
|
|
97
|
+
const audioIndex = addInput(audioArtifactFilePath); // Add audio input
|
|
98
|
+
const artifactAudioId = `${audioIndex}:a`;
|
|
99
|
+
const ffmpegContextAudioId = (() => {
|
|
100
|
+
if (filterComplexAudioIds.length > 0) {
|
|
101
|
+
const mainAudioId = "mainaudio";
|
|
102
|
+
const compositeAudioId = "composite";
|
|
103
|
+
const audioIds = filterComplexAudioIds.map((id) => `[${id}]`).join("");
|
|
104
|
+
filterComplexParts.push(`[${artifactAudioId}]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[${mainAudioId}]`);
|
|
105
|
+
filterComplexParts.push(`[${mainAudioId}]${audioIds}amix=inputs=${filterComplexAudioIds.length + 1}:duration=first:dropout_transition=2[${compositeAudioId}]`);
|
|
106
|
+
return `[${compositeAudioId}]`; // notice that we need to use [mainaudio] instead of mainaudio
|
|
107
|
+
}
|
|
108
|
+
return artifactAudioId;
|
|
109
|
+
})();
|
|
36
110
|
// Apply the filter complex for concatenation and map audio input
|
|
37
|
-
command
|
|
111
|
+
ffmpegContext.command
|
|
38
112
|
.complexFilter(filterComplexParts)
|
|
39
|
-
.
|
|
40
|
-
.outputOptions([
|
|
41
|
-
"-preset veryfast", // Faster encoding
|
|
42
|
-
"-map [v]", // Map the video stream
|
|
43
|
-
`-map ${imageCount /* + captionCount*/}:a`, // Map the audio stream (audio is the next input after all images)
|
|
44
|
-
"-c:v h264_videotoolbox", // Set video codec
|
|
45
|
-
"-threads 8",
|
|
46
|
-
"-filter_threads 8",
|
|
47
|
-
"-b:v 5M", // bitrate (only for videotoolbox)
|
|
48
|
-
"-bufsize",
|
|
49
|
-
"10M", // Add buffer size for better quality
|
|
50
|
-
"-maxrate",
|
|
51
|
-
"7M", // Maximum bitrate
|
|
52
|
-
"-r 30", // Set frame rate
|
|
53
|
-
"-pix_fmt yuv420p", // Set pixel format for better compatibility
|
|
54
|
-
])
|
|
113
|
+
.outputOptions(getOutputOption(ffmpegContextAudioId))
|
|
55
114
|
.on("start", (__cmdLine) => {
|
|
56
115
|
GraphAILogger.log("Started FFmpeg ..."); // with command:', cmdLine);
|
|
57
116
|
})
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { rgb, PDFDocument } from "pdf-lib";
|
|
4
|
+
import fontkit from "@pdf-lib/fontkit";
|
|
5
|
+
import { chunkArray, isHttp } from "../utils/utils.js";
|
|
6
|
+
import { getOutputPdfFilePath, writingMessage } from "../utils/file.js";
|
|
7
|
+
import { MulmoScriptMethods } from "../methods/index.js";
|
|
8
|
+
import { fontSize, textMargin, drawSize, wrapText } from "../utils/pdf.js";
|
|
9
|
+
const imagesPerPage = 4;
|
|
10
|
+
const offset = 10;
|
|
11
|
+
const handoutImageRatio = 0.5;
|
|
12
|
+
const readImage = async (imagePath, pdfDoc) => {
|
|
13
|
+
const imageBytes = await (async () => {
|
|
14
|
+
if (isHttp(imagePath)) {
|
|
15
|
+
const res = await fetch(imagePath);
|
|
16
|
+
const arrayBuffer = await res.arrayBuffer();
|
|
17
|
+
return Buffer.from(arrayBuffer);
|
|
18
|
+
}
|
|
19
|
+
return fs.readFileSync(imagePath);
|
|
20
|
+
})();
|
|
21
|
+
const ext = path.extname(imagePath).toLowerCase();
|
|
22
|
+
return ext === ".jpg" || ext === ".jpeg" ? await pdfDoc.embedJpg(imageBytes) : await pdfDoc.embedPng(imageBytes);
|
|
23
|
+
};
|
|
24
|
+
const pdfSlide = async (pageWidth, pageHeight, imagePaths, pdfDoc) => {
|
|
25
|
+
const cellRatio = pageHeight / pageWidth;
|
|
26
|
+
for (const imagePath of imagePaths) {
|
|
27
|
+
const image = await readImage(imagePath, pdfDoc);
|
|
28
|
+
const { width: origWidth, height: origHeight } = image.scale(1);
|
|
29
|
+
const originalRatio = origHeight / origWidth;
|
|
30
|
+
const fitWidth = originalRatio / cellRatio < 1;
|
|
31
|
+
const { drawWidth, drawHeight } = drawSize(fitWidth, pageWidth, pageHeight, origWidth, origHeight);
|
|
32
|
+
const page = pdfDoc.addPage([pageWidth, pageHeight]);
|
|
33
|
+
page.drawImage(image, {
|
|
34
|
+
x: 0,
|
|
35
|
+
y: 0,
|
|
36
|
+
width: drawWidth,
|
|
37
|
+
height: drawHeight,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
const pdfTalk = async (pageWidth, pageHeight, imagePaths, texts, pdfDoc, font) => {
|
|
42
|
+
const imageRatio = 0.7;
|
|
43
|
+
const textMargin = 8;
|
|
44
|
+
const textY = pageHeight * (1 - imageRatio) - textMargin;
|
|
45
|
+
const targetWidth = pageWidth - offset;
|
|
46
|
+
const targetHeight = pageHeight * imageRatio - offset;
|
|
47
|
+
const cellRatio = targetHeight / targetWidth;
|
|
48
|
+
for (const [index, imagePath] of imagePaths.entries()) {
|
|
49
|
+
const text = texts[index];
|
|
50
|
+
const image = await readImage(imagePath, pdfDoc);
|
|
51
|
+
const { width: origWidth, height: origHeight } = image.scale(1);
|
|
52
|
+
const originalRatio = origHeight / origWidth;
|
|
53
|
+
const fitWidth = originalRatio / cellRatio < 1;
|
|
54
|
+
const { drawWidth, drawHeight } = drawSize(fitWidth, targetWidth, targetHeight, origWidth, origHeight);
|
|
55
|
+
const x = (pageWidth - drawWidth) / 2;
|
|
56
|
+
const y = pageHeight - drawHeight - offset;
|
|
57
|
+
const page = pdfDoc.addPage([pageWidth, pageHeight]);
|
|
58
|
+
const pos = {
|
|
59
|
+
x,
|
|
60
|
+
y,
|
|
61
|
+
width: drawWidth,
|
|
62
|
+
height: drawHeight,
|
|
63
|
+
};
|
|
64
|
+
page.drawImage(image, pos);
|
|
65
|
+
page.drawRectangle({
|
|
66
|
+
...pos,
|
|
67
|
+
borderColor: rgb(0, 0, 0),
|
|
68
|
+
borderWidth: 1,
|
|
69
|
+
});
|
|
70
|
+
const lines = wrapText(text, font, fontSize, pageWidth - textMargin * 2);
|
|
71
|
+
for (const [index, line] of lines.entries()) {
|
|
72
|
+
page.drawText(line, {
|
|
73
|
+
x: textMargin,
|
|
74
|
+
y: textY - fontSize - (fontSize + 2) * index,
|
|
75
|
+
size: fontSize,
|
|
76
|
+
color: rgb(0, 0, 0),
|
|
77
|
+
maxWidth: pageWidth - 2 * textMargin,
|
|
78
|
+
font,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
const pdfHandout = async (pageWidth, pageHeight, imagePaths, texts, pdfDoc, font, isLandscapeImage) => {
|
|
84
|
+
const cellRatio = (pageHeight / imagesPerPage - offset) / (pageWidth * handoutImageRatio - offset);
|
|
85
|
+
let index = 0;
|
|
86
|
+
for (const chunkPaths of chunkArray(imagePaths, imagesPerPage)) {
|
|
87
|
+
const page = pdfDoc.addPage([pageWidth, pageHeight]);
|
|
88
|
+
for (let i = 0; i < chunkPaths.length; i++) {
|
|
89
|
+
const imagePath = chunkPaths[i];
|
|
90
|
+
const image = await readImage(imagePath, pdfDoc);
|
|
91
|
+
const { width: origWidth, height: origHeight } = image.scale(1);
|
|
92
|
+
const originalRatio = origHeight / origWidth;
|
|
93
|
+
const fitWidth = originalRatio / cellRatio < 1;
|
|
94
|
+
const pos = (() => {
|
|
95
|
+
if (isLandscapeImage) {
|
|
96
|
+
const cellHeight = pageHeight / imagesPerPage - offset;
|
|
97
|
+
const { drawWidth, drawHeight } = drawSize(fitWidth, (pageWidth - offset) * handoutImageRatio, cellHeight - offset, origWidth, origHeight);
|
|
98
|
+
const x = offset;
|
|
99
|
+
const y = pageHeight - (i + 1) * cellHeight + (cellHeight - drawHeight) * handoutImageRatio;
|
|
100
|
+
return {
|
|
101
|
+
x,
|
|
102
|
+
y,
|
|
103
|
+
width: drawWidth,
|
|
104
|
+
height: drawHeight,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
const cellWidth = pageWidth / imagesPerPage;
|
|
109
|
+
const { drawWidth, drawHeight } = drawSize(fitWidth, cellWidth - offset, (pageHeight - offset) * handoutImageRatio, origWidth, origHeight);
|
|
110
|
+
const x = pageWidth - (imagesPerPage - i) * cellWidth + (cellWidth - drawWidth) * handoutImageRatio;
|
|
111
|
+
const y = pageHeight - drawHeight - offset;
|
|
112
|
+
return {
|
|
113
|
+
x,
|
|
114
|
+
y,
|
|
115
|
+
width: drawWidth,
|
|
116
|
+
height: drawHeight,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
})();
|
|
120
|
+
page.drawRectangle({
|
|
121
|
+
...pos,
|
|
122
|
+
borderColor: rgb(0, 0, 0),
|
|
123
|
+
borderWidth: 1,
|
|
124
|
+
});
|
|
125
|
+
page.drawImage(image, pos);
|
|
126
|
+
if (isLandscapeImage) {
|
|
127
|
+
const lines = wrapText(texts[index], font, fontSize, pos.width - textMargin * 2);
|
|
128
|
+
for (const [index, line] of lines.entries()) {
|
|
129
|
+
page.drawText(line, {
|
|
130
|
+
...pos,
|
|
131
|
+
x: pos.x + pos.width + textMargin,
|
|
132
|
+
y: pos.y + pos.height - fontSize - (fontSize + 2) * index,
|
|
133
|
+
size: fontSize,
|
|
134
|
+
font,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
/*
|
|
138
|
+
page.drawRectangle({
|
|
139
|
+
...pos,
|
|
140
|
+
x: pos.x + pos.width ,
|
|
141
|
+
borderColor: rgb(0, 0, 0),
|
|
142
|
+
borderWidth: 1,
|
|
143
|
+
});
|
|
144
|
+
*/
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
const lines = wrapText(texts[index], font, fontSize, pos.width - textMargin * 2);
|
|
148
|
+
for (const [index, line] of lines.entries()) {
|
|
149
|
+
page.drawText(line, {
|
|
150
|
+
...pos,
|
|
151
|
+
x: pos.x,
|
|
152
|
+
y: textMargin + pos.height - fontSize - (fontSize + textMargin) * index - 2 * textMargin,
|
|
153
|
+
size: fontSize,
|
|
154
|
+
font,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
/*
|
|
158
|
+
page.drawRectangle({
|
|
159
|
+
...pos,
|
|
160
|
+
x: pos.x,
|
|
161
|
+
y: textMargin,
|
|
162
|
+
height: pos.height - 2 * textMargin,
|
|
163
|
+
borderColor: rgb(0, 0, 0),
|
|
164
|
+
borderWidth: 1,
|
|
165
|
+
});
|
|
166
|
+
*/
|
|
167
|
+
}
|
|
168
|
+
index = index + 1;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
const outputSize = (pdfSize, isLandscapeImage, isRotate) => {
|
|
173
|
+
if (pdfSize === "a4") {
|
|
174
|
+
if (isLandscapeImage !== isRotate) {
|
|
175
|
+
return { width: 841.89, height: 595.28 };
|
|
176
|
+
}
|
|
177
|
+
return { width: 595.28, height: 841.89 };
|
|
178
|
+
}
|
|
179
|
+
// letter
|
|
180
|
+
if (isLandscapeImage !== isRotate) {
|
|
181
|
+
return { width: 792, height: 612 };
|
|
182
|
+
}
|
|
183
|
+
return { width: 612, height: 792 };
|
|
184
|
+
};
|
|
185
|
+
export const pdf = async (context, pdfMode, pdfSize) => {
|
|
186
|
+
const { studio, fileDirs } = context;
|
|
187
|
+
const { outDirPath } = fileDirs;
|
|
188
|
+
const { width: imageWidth, height: imageHeight } = MulmoScriptMethods.getCanvasSize(studio.script);
|
|
189
|
+
const isLandscapeImage = imageWidth > imageHeight;
|
|
190
|
+
const isRotate = pdfMode === "handout";
|
|
191
|
+
const { width: pageWidth, height: pageHeight } = outputSize(pdfSize, isLandscapeImage, isRotate);
|
|
192
|
+
const imagePaths = studio.beats.map((beat) => beat.imageFile);
|
|
193
|
+
const texts = studio.script.beats.map((beat) => beat.text);
|
|
194
|
+
const outputPdfPath = getOutputPdfFilePath(outDirPath, studio.filename, pdfMode);
|
|
195
|
+
const pdfDoc = await PDFDocument.create();
|
|
196
|
+
pdfDoc.registerFontkit(fontkit);
|
|
197
|
+
const fontBytes = fs.readFileSync("assets/font/NotoSansJP-Regular.ttf");
|
|
198
|
+
const customFont = await pdfDoc.embedFont(fontBytes);
|
|
199
|
+
if (pdfMode === "handout") {
|
|
200
|
+
await pdfHandout(pageWidth, pageHeight, imagePaths, texts, pdfDoc, customFont, isLandscapeImage);
|
|
201
|
+
}
|
|
202
|
+
if (pdfMode === "slide") {
|
|
203
|
+
await pdfSlide(pageWidth, pageHeight, imagePaths, pdfDoc);
|
|
204
|
+
}
|
|
205
|
+
if (pdfMode === "talk") {
|
|
206
|
+
await pdfTalk(pageWidth, pageHeight, imagePaths, texts, pdfDoc, customFont);
|
|
207
|
+
}
|
|
208
|
+
const pdfBytes = await pdfDoc.save();
|
|
209
|
+
fs.writeFileSync(outputPdfPath, pdfBytes);
|
|
210
|
+
writingMessage(outputPdfPath);
|
|
211
|
+
};
|
package/lib/actions/translate.js
CHANGED
|
@@ -32,7 +32,8 @@ const translateGraph = {
|
|
|
32
32
|
agent: "mapAgent",
|
|
33
33
|
inputs: {
|
|
34
34
|
targetLangs: ":targetLangs",
|
|
35
|
-
|
|
35
|
+
studio: ":studio",
|
|
36
|
+
rows: ":studio.script.beats",
|
|
36
37
|
lang: ":lang",
|
|
37
38
|
},
|
|
38
39
|
params: {
|
|
@@ -42,12 +43,23 @@ const translateGraph = {
|
|
|
42
43
|
graph: {
|
|
43
44
|
version: 0.5,
|
|
44
45
|
nodes: {
|
|
46
|
+
studioBeat: {
|
|
47
|
+
agent: (namedInputs) => {
|
|
48
|
+
return namedInputs.rows[namedInputs.index];
|
|
49
|
+
},
|
|
50
|
+
inputs: {
|
|
51
|
+
index: ":__mapIndex",
|
|
52
|
+
rows: ":studio.beats",
|
|
53
|
+
},
|
|
54
|
+
},
|
|
45
55
|
preprocessBeats: {
|
|
46
56
|
agent: "mapAgent",
|
|
47
57
|
inputs: {
|
|
48
58
|
beat: ":beat",
|
|
59
|
+
studioBeat: ":studioBeat",
|
|
49
60
|
rows: ":targetLangs",
|
|
50
61
|
lang: ":lang.text",
|
|
62
|
+
studio: ":studio",
|
|
51
63
|
},
|
|
52
64
|
params: {
|
|
53
65
|
compositeResult: true,
|
|
@@ -60,6 +72,7 @@ const translateGraph = {
|
|
|
60
72
|
inputs: {
|
|
61
73
|
targetLang: ":targetLang",
|
|
62
74
|
beat: ":beat",
|
|
75
|
+
studioBeat: ":studioBeat",
|
|
63
76
|
lang: ":lang",
|
|
64
77
|
system: "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).",
|
|
65
78
|
prompt: ["## Original Language", ":lang", "", "## Language", ":targetLang", "", "## Target", ":beat.text"],
|
|
@@ -138,7 +151,7 @@ const translateGraph = {
|
|
|
138
151
|
isResult: true,
|
|
139
152
|
agent: "mergeObjectAgent",
|
|
140
153
|
inputs: {
|
|
141
|
-
items: [":
|
|
154
|
+
items: [":studioBeat", { multiLingualTexts: ":mergeLocalizedText" }],
|
|
142
155
|
},
|
|
143
156
|
},
|
|
144
157
|
},
|
|
@@ -156,14 +169,14 @@ const translateGraph = {
|
|
|
156
169
|
};
|
|
157
170
|
const localizedTextCacheAgentFilter = async (context, next) => {
|
|
158
171
|
const { namedInputs } = context;
|
|
159
|
-
const { targetLang, beat, lang } = namedInputs;
|
|
172
|
+
const { targetLang, beat, lang, studioBeat } = namedInputs;
|
|
160
173
|
// The original text is unchanged and the target language text is present
|
|
161
|
-
if (
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
return { text:
|
|
174
|
+
if (studioBeat.multiLingualTexts &&
|
|
175
|
+
studioBeat.multiLingualTexts[lang] &&
|
|
176
|
+
studioBeat.multiLingualTexts[lang].text === beat.text &&
|
|
177
|
+
studioBeat.multiLingualTexts[targetLang] &&
|
|
178
|
+
studioBeat.multiLingualTexts[targetLang].text) {
|
|
179
|
+
return { text: studioBeat.multiLingualTexts[targetLang].text };
|
|
167
180
|
}
|
|
168
181
|
// same language
|
|
169
182
|
if (targetLang === lang) {
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import ffmpeg from "fluent-ffmpeg";
|
|
3
|
-
import { silentPath, silentLastPath
|
|
4
|
-
import { MulmoStudioContextMethods } from "../methods/index.js";
|
|
3
|
+
import { silentPath, silentLastPath } from "../utils/file.js";
|
|
5
4
|
const combineAudioFilesAgent = async ({ namedInputs }) => {
|
|
6
5
|
const { context, combinedFileName, audioDirPath } = namedInputs;
|
|
7
6
|
const command = ffmpeg();
|
|
8
|
-
const getDuration = (filePath,
|
|
7
|
+
const getDuration = (filePath, isLastGap) => {
|
|
9
8
|
return new Promise((resolve, reject) => {
|
|
10
9
|
ffmpeg.ffprobe(filePath, (err, metadata) => {
|
|
11
10
|
if (err) {
|
|
@@ -13,30 +12,22 @@ const combineAudioFilesAgent = async ({ namedInputs }) => {
|
|
|
13
12
|
reject(err);
|
|
14
13
|
}
|
|
15
14
|
else {
|
|
16
|
-
|
|
15
|
+
// TODO: Remove hard-coded 0.8 and 0.3
|
|
16
|
+
resolve(metadata.format.duration + (isLastGap ? 0.8 : 0.3));
|
|
17
17
|
}
|
|
18
18
|
});
|
|
19
19
|
});
|
|
20
20
|
};
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
21
|
+
await Promise.all(context.studio.beats.map(async (studioBeat, index) => {
|
|
22
|
+
const isLastGap = index === context.studio.beats.length - 2;
|
|
23
|
+
if (studioBeat.audioFile) {
|
|
24
|
+
command.input(studioBeat.audioFile);
|
|
25
|
+
command.input(isLastGap ? silentLastPath : silentPath);
|
|
26
|
+
studioBeat.duration = await getDuration(studioBeat.audioFile, isLastGap);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
GraphAILogger.error("Missing studioBeat.audioFile:", index);
|
|
30
30
|
}
|
|
31
|
-
return getAudioSegmentFilePath(audioDirPath, context.studio.filename, mulmoBeat.audioFile ?? "");
|
|
32
|
-
};
|
|
33
|
-
await Promise.all(context.studio.beats.map(async (mulmoBeat, index) => {
|
|
34
|
-
const filePath = resolveAudioFilePath(context, mulmoBeat, audioDirPath);
|
|
35
|
-
const isLast = index === context.studio.beats.length - 2;
|
|
36
|
-
command.input(filePath);
|
|
37
|
-
command.input(isLast ? silentLastPath : silentPath);
|
|
38
|
-
// Measure and log the timestamp of each section
|
|
39
|
-
context.studio.beats[index]["duration"] = await getDuration(filePath, isLast);
|
|
40
31
|
}));
|
|
41
32
|
await new Promise((resolve, reject) => {
|
|
42
33
|
command
|
package/lib/cli/args.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const
|
|
1
|
+
export declare const getArgs: () => {
|
|
2
2
|
[x: string]: unknown;
|
|
3
3
|
v: boolean;
|
|
4
4
|
o: string | undefined;
|
|
@@ -6,6 +6,8 @@ export declare const args: {
|
|
|
6
6
|
a: string | undefined;
|
|
7
7
|
i: string | undefined;
|
|
8
8
|
f: boolean;
|
|
9
|
+
pdf_mode: string;
|
|
10
|
+
pdf_size: string;
|
|
9
11
|
_: (string | number)[];
|
|
10
12
|
$0: string;
|
|
11
13
|
};
|
package/lib/cli/args.js
CHANGED
|
@@ -1,40 +1,55 @@
|
|
|
1
1
|
import yargs from "yargs";
|
|
2
2
|
import { hideBin } from "yargs/helpers";
|
|
3
3
|
import { commonOptions } from "./common.js";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
.
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
.option("i", {
|
|
13
|
-
alias: "imagedir",
|
|
14
|
-
description: "image dir",
|
|
15
|
-
demandOption: false,
|
|
16
|
-
type: "string",
|
|
17
|
-
})
|
|
18
|
-
.option("f", {
|
|
19
|
-
alias: "force",
|
|
20
|
-
description: "force generate",
|
|
21
|
-
demandOption: false,
|
|
22
|
-
default: false,
|
|
23
|
-
type: "boolean",
|
|
24
|
-
})
|
|
25
|
-
.command("$0 <action> <file>", "Run mulmocast", (yargs) => {
|
|
26
|
-
return yargs
|
|
27
|
-
.positional("action", {
|
|
28
|
-
describe: "action to perform",
|
|
29
|
-
choices: ["translate", "audio", "images", "movie", "preprocess"],
|
|
4
|
+
import { pdf_modes, pdf_sizes } from "../utils/const.js";
|
|
5
|
+
export const getArgs = () => {
|
|
6
|
+
return commonOptions(yargs(hideBin(process.argv)))
|
|
7
|
+
.scriptName("mulmo")
|
|
8
|
+
.option("a", {
|
|
9
|
+
alias: "audiodir",
|
|
10
|
+
description: "audio dir",
|
|
11
|
+
demandOption: false,
|
|
30
12
|
type: "string",
|
|
31
13
|
})
|
|
32
|
-
.
|
|
33
|
-
|
|
14
|
+
.option("i", {
|
|
15
|
+
alias: "imagedir",
|
|
16
|
+
description: "image dir",
|
|
17
|
+
demandOption: false,
|
|
34
18
|
type: "string",
|
|
35
|
-
})
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
19
|
+
})
|
|
20
|
+
.option("f", {
|
|
21
|
+
alias: "force",
|
|
22
|
+
description: "force generate",
|
|
23
|
+
demandOption: false,
|
|
24
|
+
default: false,
|
|
25
|
+
type: "boolean",
|
|
26
|
+
})
|
|
27
|
+
.option("pdf_mode", {
|
|
28
|
+
description: "pdf mode",
|
|
29
|
+
demandOption: false,
|
|
30
|
+
choices: pdf_modes,
|
|
31
|
+
type: "string",
|
|
32
|
+
default: "slide",
|
|
33
|
+
})
|
|
34
|
+
.option("pdf_size", {
|
|
35
|
+
choices: pdf_sizes,
|
|
36
|
+
default: "letter",
|
|
37
|
+
describe: "PDF paper size (default: letter for US standard)",
|
|
38
|
+
})
|
|
39
|
+
.command("$0 <action> <file>", "Run mulmocast", (yargs) => {
|
|
40
|
+
return yargs
|
|
41
|
+
.positional("action", {
|
|
42
|
+
describe: "action to perform",
|
|
43
|
+
choices: ["translate", "audio", "images", "movie", "pdf", "preprocess"],
|
|
44
|
+
type: "string",
|
|
45
|
+
})
|
|
46
|
+
.positional("file", {
|
|
47
|
+
describe: "Mulmo Script File",
|
|
48
|
+
type: "string",
|
|
49
|
+
});
|
|
50
|
+
})
|
|
51
|
+
.strict()
|
|
52
|
+
.help()
|
|
53
|
+
.alias("help", "h")
|
|
54
|
+
.parseSync();
|
|
55
|
+
};
|
package/lib/cli/cli.d.ts
CHANGED
|
@@ -1,2 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import "dotenv/config";
|
|
3
|
+
export declare const getFileObject: (_args: {
|
|
4
|
+
[x: string]: unknown;
|
|
5
|
+
}) => {
|
|
6
|
+
baseDirPath: string;
|
|
7
|
+
mulmoFilePath: string;
|
|
8
|
+
mulmoFileDirPath: string;
|
|
9
|
+
outDirPath: string;
|
|
10
|
+
imageDirPath: string;
|
|
11
|
+
audioDirPath: string;
|
|
12
|
+
isHttpPath: boolean;
|
|
13
|
+
fileOrUrl: string;
|
|
14
|
+
outputStudioFilePath: string;
|
|
15
|
+
fileName: string;
|
|
16
|
+
};
|
|
17
|
+
export declare const main: () => Promise<void>;
|