mulmocast 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/templates/text_and_image.json +6 -0
- package/assets/templates/text_only.json +6 -0
- package/lib/actions/audio.d.ts +4 -2
- package/lib/actions/audio.js +89 -48
- package/lib/actions/captions.d.ts +1 -1
- package/lib/actions/captions.js +17 -14
- package/lib/actions/images.d.ts +6 -3
- package/lib/actions/images.js +64 -39
- package/lib/actions/movie.js +19 -19
- package/lib/actions/pdf.js +3 -4
- package/lib/actions/translate.js +11 -11
- package/lib/agents/add_bgm_agent.js +3 -3
- package/lib/agents/combine_audio_files_agent.js +88 -42
- package/lib/agents/index.d.ts +2 -1
- package/lib/agents/index.js +2 -1
- package/lib/agents/tavily_agent.d.ts +15 -0
- package/lib/agents/tavily_agent.js +130 -0
- package/lib/cli/commands/audio/builder.d.ts +2 -0
- package/lib/cli/commands/image/builder.d.ts +2 -0
- package/lib/cli/commands/movie/builder.d.ts +2 -0
- package/lib/cli/commands/movie/handler.js +1 -6
- package/lib/cli/commands/pdf/builder.d.ts +2 -0
- package/lib/cli/commands/translate/builder.d.ts +2 -0
- package/lib/cli/common.d.ts +2 -0
- package/lib/cli/common.js +6 -0
- package/lib/cli/helpers.d.ts +7 -1
- package/lib/cli/helpers.js +30 -3
- package/lib/methods/index.d.ts +1 -1
- package/lib/methods/index.js +1 -1
- package/lib/methods/mulmo_presentation_style.d.ts +14 -0
- package/lib/methods/mulmo_presentation_style.js +70 -0
- package/lib/methods/mulmo_studio_context.d.ts +17 -0
- package/lib/methods/mulmo_studio_context.js +30 -2
- package/lib/tools/deep_research.d.ts +2 -0
- package/lib/tools/deep_research.js +265 -0
- package/lib/types/index.d.ts +0 -1
- package/lib/types/index.js +0 -1
- package/lib/types/schema.d.ts +101 -55
- package/lib/types/schema.js +3 -3
- package/lib/types/type.d.ts +5 -1
- package/lib/utils/ffmpeg_utils.d.ts +1 -0
- package/lib/utils/ffmpeg_utils.js +10 -0
- package/lib/utils/file.d.ts +7 -4
- package/lib/utils/file.js +24 -12
- package/lib/utils/preprocess.d.ts +0 -9
- package/lib/utils/preprocess.js +4 -10
- package/lib/utils/prompt.d.ts +3 -0
- package/lib/utils/prompt.js +52 -0
- package/package.json +11 -10
- package/assets/music/StarsBeyondEx.mp3 +0 -0
package/lib/actions/movie.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { GraphAILogger, assert } from "graphai";
|
|
2
2
|
import { mulmoTransitionSchema } from "../types/index.js";
|
|
3
|
-
import {
|
|
3
|
+
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
4
4
|
import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage } from "../utils/file.js";
|
|
5
5
|
import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput } from "../utils/ffmpeg_utils.js";
|
|
6
6
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
@@ -59,22 +59,22 @@ const getOutputOption = (audioId, videoId) => {
|
|
|
59
59
|
"-b:a 128k", // Audio bitrate
|
|
60
60
|
];
|
|
61
61
|
};
|
|
62
|
-
const createVideo = async (audioArtifactFilePath, outputVideoPath,
|
|
62
|
+
const createVideo = async (audioArtifactFilePath, outputVideoPath, context, caption) => {
|
|
63
63
|
const start = performance.now();
|
|
64
64
|
const ffmpegContext = FfmpegContextInit();
|
|
65
|
-
const missingIndex = studio.beats.findIndex((beat) => !beat.imageFile && !beat.movieFile);
|
|
65
|
+
const missingIndex = context.studio.beats.findIndex((beat) => !beat.imageFile && !beat.movieFile);
|
|
66
66
|
if (missingIndex !== -1) {
|
|
67
67
|
GraphAILogger.info(`ERROR: beat.imageFile or beat.movieFile is not set on beat ${missingIndex}.`);
|
|
68
68
|
return false;
|
|
69
69
|
}
|
|
70
|
-
const canvasInfo =
|
|
70
|
+
const canvasInfo = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
|
|
71
71
|
// Add each image input
|
|
72
72
|
const filterComplexVideoIds = [];
|
|
73
73
|
const filterComplexAudioIds = [];
|
|
74
74
|
const transitionVideoIds = [];
|
|
75
75
|
const beatTimestamps = [];
|
|
76
|
-
studio.beats.reduce((timestamp, studioBeat, index) => {
|
|
77
|
-
const beat = studio.script.beats[index];
|
|
76
|
+
context.studio.beats.reduce((timestamp, studioBeat, index) => {
|
|
77
|
+
const beat = context.studio.script.beats[index];
|
|
78
78
|
const sourceFile = studioBeat.movieFile ?? studioBeat.imageFile;
|
|
79
79
|
if (!sourceFile) {
|
|
80
80
|
throw new Error(`studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
|
|
@@ -83,14 +83,14 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
83
83
|
throw new Error(`studioBeat.duration is not set: index=${index}`);
|
|
84
84
|
}
|
|
85
85
|
const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
|
|
86
|
-
const mediaType = studioBeat.movieFile ? "movie" :
|
|
86
|
+
const mediaType = studioBeat.movieFile ? "movie" : MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat);
|
|
87
87
|
const extraPadding = (() => {
|
|
88
88
|
// We need to consider only intro and outro padding because the other paddings were already added to the beat.duration
|
|
89
89
|
if (index === 0) {
|
|
90
|
-
return
|
|
90
|
+
return context.presentationStyle.audioParams.introPadding;
|
|
91
91
|
}
|
|
92
|
-
else if (index === studio.beats.length - 1) {
|
|
93
|
-
return
|
|
92
|
+
else if (index === context.studio.beats.length - 1) {
|
|
93
|
+
return context.presentationStyle.audioParams.outroPadding;
|
|
94
94
|
}
|
|
95
95
|
return 0;
|
|
96
96
|
})();
|
|
@@ -106,7 +106,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
106
106
|
else {
|
|
107
107
|
filterComplexVideoIds.push(videoId);
|
|
108
108
|
}
|
|
109
|
-
if (
|
|
109
|
+
if (context.presentationStyle.movieParams?.transition && index < context.studio.beats.length - 1) {
|
|
110
110
|
const sourceId = filterComplexVideoIds.pop();
|
|
111
111
|
ffmpegContext.filterComplex.push(`[${sourceId}]split=2[${sourceId}_0][${sourceId}_1]`);
|
|
112
112
|
filterComplexVideoIds.push(`${sourceId}_0`);
|
|
@@ -127,16 +127,16 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
127
127
|
beatTimestamps.push(timestamp);
|
|
128
128
|
return timestamp + duration;
|
|
129
129
|
}, 0);
|
|
130
|
-
assert(filterComplexVideoIds.length === studio.beats.length, "videoIds.length !== studio.beats.length");
|
|
131
|
-
assert(beatTimestamps.length === studio.beats.length, "beatTimestamps.length !== studio.beats.length");
|
|
130
|
+
assert(filterComplexVideoIds.length === context.studio.beats.length, "videoIds.length !== studio.beats.length");
|
|
131
|
+
assert(beatTimestamps.length === context.studio.beats.length, "beatTimestamps.length !== studio.beats.length");
|
|
132
132
|
// console.log("*** images", images.audioIds);
|
|
133
133
|
// Concatenate the trimmed images
|
|
134
134
|
const concatVideoId = "concat_video";
|
|
135
|
-
ffmpegContext.filterComplex.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${studio.beats.length}:v=1:a=0[${concatVideoId}]`);
|
|
135
|
+
ffmpegContext.filterComplex.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${context.studio.beats.length}:v=1:a=0[${concatVideoId}]`);
|
|
136
136
|
// Add tranditions if needed
|
|
137
137
|
const mixedVideoId = (() => {
|
|
138
|
-
if (
|
|
139
|
-
const transition = mulmoTransitionSchema.parse(
|
|
138
|
+
if (context.presentationStyle.movieParams?.transition && transitionVideoIds.length > 0) {
|
|
139
|
+
const transition = mulmoTransitionSchema.parse(context.presentationStyle.movieParams.transition);
|
|
140
140
|
return transitionVideoIds.reduce((acc, transitionVideoId, index) => {
|
|
141
141
|
const transitionStartTime = beatTimestamps[index + 1] - 0.05; // 0.05 is to avoid flickering
|
|
142
142
|
const processedVideoId = `${transitionVideoId}_f`;
|
|
@@ -166,8 +166,8 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
|
|
|
166
166
|
await FfmpegContextGenerateOutput(ffmpegContext, outputVideoPath, getOutputOption(ffmpegContextAudioId, mixedVideoId));
|
|
167
167
|
const end = performance.now();
|
|
168
168
|
GraphAILogger.info(`Video created successfully! ${Math.round(end - start) / 1000} sec`);
|
|
169
|
-
GraphAILogger.info(studio.script.title);
|
|
170
|
-
GraphAILogger.info((studio.script.references ?? []).map((reference) => `${reference.title} (${reference.url})`).join("\n"));
|
|
169
|
+
GraphAILogger.info(context.studio.script.title);
|
|
170
|
+
GraphAILogger.info((context.studio.script.references ?? []).map((reference) => `${reference.title} (${reference.url})`).join("\n"));
|
|
171
171
|
return true;
|
|
172
172
|
};
|
|
173
173
|
export const movieFilePath = (context) => {
|
|
@@ -181,7 +181,7 @@ export const movie = async (context) => {
|
|
|
181
181
|
const { outDirPath } = fileDirs;
|
|
182
182
|
const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, studio.filename);
|
|
183
183
|
const outputVideoPath = movieFilePath(context);
|
|
184
|
-
if (await createVideo(audioArtifactFilePath, outputVideoPath,
|
|
184
|
+
if (await createVideo(audioArtifactFilePath, outputVideoPath, context, caption)) {
|
|
185
185
|
writingMessage(outputVideoPath);
|
|
186
186
|
}
|
|
187
187
|
}
|
package/lib/actions/pdf.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import puppeteer from "puppeteer";
|
|
4
|
-
import {
|
|
4
|
+
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
5
5
|
import { localizedText, isHttp } from "../utils/utils.js";
|
|
6
6
|
import { getOutputPdfFilePath, writingMessage, getHTMLFile } from "../utils/file.js";
|
|
7
7
|
import { interpolate } from "../utils/markdown.js";
|
|
@@ -95,9 +95,8 @@ const getHandoutTemplateData = (isLandscapeImage) => ({
|
|
|
95
95
|
item_flex: isLandscapeImage ? "flex: 1;" : "",
|
|
96
96
|
});
|
|
97
97
|
const generatePDFHTML = async (context, pdfMode, pdfSize) => {
|
|
98
|
-
const { studio, lang = "en" } = context;
|
|
99
|
-
const {
|
|
100
|
-
const { width: imageWidth, height: imageHeight } = MulmoScriptMethods.getCanvasSize(studio.script);
|
|
98
|
+
const { studio, multiLingual, lang = "en" } = context;
|
|
99
|
+
const { width: imageWidth, height: imageHeight } = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
|
|
101
100
|
const isLandscapeImage = imageWidth > imageHeight;
|
|
102
101
|
const imagePaths = studio.beats.map((beat) => beat.imageFile);
|
|
103
102
|
const texts = studio.script.beats.map((beat, index) => localizedText(beat, multiLingual?.[index], lang));
|
package/lib/actions/translate.js
CHANGED
|
@@ -4,7 +4,7 @@ import * as agents from "@graphai/vanilla";
|
|
|
4
4
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
5
5
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
6
6
|
import { recursiveSplitJa, replacementsJa, replacePairsJa } from "../utils/string.js";
|
|
7
|
-
import {
|
|
7
|
+
import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
|
|
8
8
|
import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
|
|
9
9
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
10
10
|
const vanillaAgents = agents.default ?? agents;
|
|
@@ -14,7 +14,7 @@ const translateGraph = {
|
|
|
14
14
|
context: {},
|
|
15
15
|
defaultLang: {},
|
|
16
16
|
outDirPath: {},
|
|
17
|
-
|
|
17
|
+
outputMultilingualFilePath: {},
|
|
18
18
|
lang: {
|
|
19
19
|
agent: "stringUpdateTextAgent",
|
|
20
20
|
inputs: {
|
|
@@ -27,7 +27,7 @@ const translateGraph = {
|
|
|
27
27
|
isResult: true,
|
|
28
28
|
agent: "mergeObjectAgent",
|
|
29
29
|
inputs: {
|
|
30
|
-
items: [
|
|
30
|
+
items: [{ multiLingual: ":beatsMap.mergeMultiLingualData" }],
|
|
31
31
|
},
|
|
32
32
|
},
|
|
33
33
|
beatsMap: {
|
|
@@ -52,7 +52,7 @@ const translateGraph = {
|
|
|
52
52
|
},
|
|
53
53
|
inputs: {
|
|
54
54
|
index: ":__mapIndex",
|
|
55
|
-
rows: ":context.
|
|
55
|
+
rows: ":context.multiLingual",
|
|
56
56
|
},
|
|
57
57
|
},
|
|
58
58
|
preprocessMultiLingual: {
|
|
@@ -163,12 +163,12 @@ const translateGraph = {
|
|
|
163
163
|
},
|
|
164
164
|
},
|
|
165
165
|
},
|
|
166
|
-
|
|
166
|
+
writeOutput: {
|
|
167
167
|
// console: { before: true },
|
|
168
168
|
agent: "fileWriteAgent",
|
|
169
169
|
inputs: {
|
|
170
|
-
file: ":
|
|
171
|
-
text: ":mergeStudioResult.toJSON()",
|
|
170
|
+
file: ":outputMultilingualFilePath",
|
|
171
|
+
text: ":mergeStudioResult.multiLingual.toJSON()",
|
|
172
172
|
},
|
|
173
173
|
},
|
|
174
174
|
},
|
|
@@ -213,7 +213,7 @@ export const translate = async (context, callbacks) => {
|
|
|
213
213
|
MulmoStudioContextMethods.setSessionState(context, "multiLingual", true);
|
|
214
214
|
const { studio, fileDirs } = context;
|
|
215
215
|
const { outDirPath } = fileDirs;
|
|
216
|
-
const
|
|
216
|
+
const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, studio.filename);
|
|
217
217
|
mkdir(outDirPath);
|
|
218
218
|
assert(!!process.env.OPENAI_API_KEY, "The OPENAI_API_KEY environment variable is missing or empty");
|
|
219
219
|
const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters });
|
|
@@ -221,16 +221,16 @@ export const translate = async (context, callbacks) => {
|
|
|
221
221
|
graph.injectValue("defaultLang", defaultLang);
|
|
222
222
|
graph.injectValue("targetLangs", targetLangs);
|
|
223
223
|
graph.injectValue("outDirPath", outDirPath);
|
|
224
|
-
graph.injectValue("
|
|
224
|
+
graph.injectValue("outputMultilingualFilePath", outputMultilingualFilePath);
|
|
225
225
|
if (callbacks) {
|
|
226
226
|
callbacks.forEach((callback) => {
|
|
227
227
|
graph.registerCallback(callback);
|
|
228
228
|
});
|
|
229
229
|
}
|
|
230
230
|
const results = await graph.run();
|
|
231
|
-
writingMessage(
|
|
231
|
+
writingMessage(outputMultilingualFilePath);
|
|
232
232
|
if (results.mergeStudioResult) {
|
|
233
|
-
context.
|
|
233
|
+
context.multiLingual = results.mergeStudioResult.multiLingual;
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
236
|
finally {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextGenerateOutput, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
|
|
3
3
|
const addBGMAgent = async ({ namedInputs, params, }) => {
|
|
4
|
-
const { voiceFile, outputFile,
|
|
4
|
+
const { voiceFile, outputFile, context } = namedInputs;
|
|
5
5
|
const { musicFile } = params;
|
|
6
6
|
const speechDuration = await ffmpegGetMediaDuration(voiceFile);
|
|
7
|
-
const introPadding =
|
|
8
|
-
const outroPadding =
|
|
7
|
+
const introPadding = context.presentationStyle.audioParams.introPadding;
|
|
8
|
+
const outroPadding = context.presentationStyle.audioParams.outroPadding;
|
|
9
9
|
const totalDuration = speechDuration + introPadding + outroPadding;
|
|
10
10
|
GraphAILogger.log("totalDucation:", speechDuration, totalDuration);
|
|
11
11
|
const ffmpegContext = FfmpegContextInit();
|
|
@@ -1,6 +1,35 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { assert } from "graphai";
|
|
2
2
|
import { silent60secPath } from "../utils/file.js";
|
|
3
3
|
import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
|
|
4
|
+
const getMovieDulation = async (beat) => {
|
|
5
|
+
if (beat.image?.type === "movie" && (beat.image.source.kind === "url" || beat.image.source.kind === "path")) {
|
|
6
|
+
const pathOrUrl = beat.image.source.kind === "url" ? beat.image.source.url : beat.image.source.path;
|
|
7
|
+
return await ffmpegGetMediaDuration(pathOrUrl);
|
|
8
|
+
}
|
|
9
|
+
return 0;
|
|
10
|
+
};
|
|
11
|
+
const getPadding = (context, beat, index) => {
|
|
12
|
+
if (beat.audioParams?.padding !== undefined) {
|
|
13
|
+
return beat.audioParams.padding;
|
|
14
|
+
}
|
|
15
|
+
if (index === context.studio.beats.length - 1) {
|
|
16
|
+
return 0;
|
|
17
|
+
}
|
|
18
|
+
const isClosingGap = index === context.studio.beats.length - 2;
|
|
19
|
+
return isClosingGap ? context.presentationStyle.audioParams.closingPadding : context.presentationStyle.audioParams.padding;
|
|
20
|
+
};
|
|
21
|
+
const getTotalPadding = (padding, movieDuration, audioDuration, duration, canSpillover = false) => {
|
|
22
|
+
if (movieDuration > 0) {
|
|
23
|
+
return padding + (movieDuration - audioDuration);
|
|
24
|
+
}
|
|
25
|
+
else if (duration && duration > audioDuration) {
|
|
26
|
+
return padding + (duration - audioDuration);
|
|
27
|
+
}
|
|
28
|
+
else if (canSpillover && duration && audioDuration > duration) {
|
|
29
|
+
return duration - audioDuration; // negative value to indicate that there is a spill over.
|
|
30
|
+
}
|
|
31
|
+
return padding;
|
|
32
|
+
};
|
|
4
33
|
const combineAudioFilesAgent = async ({ namedInputs, }) => {
|
|
5
34
|
const { context, combinedFileName } = namedInputs;
|
|
6
35
|
const ffmpegContext = FfmpegContextInit();
|
|
@@ -8,64 +37,81 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
|
|
|
8
37
|
// We cannot reuse longSilentId. We need to explicitly split it for each beat.
|
|
9
38
|
const silentIds = context.studio.beats.map((_, index) => `[ls_${index}]`);
|
|
10
39
|
ffmpegContext.filterComplex.push(`${longSilentId}asplit=${silentIds.length}${silentIds.join("")}`);
|
|
11
|
-
|
|
40
|
+
// First, get the audio durations of all beats, taking advantage of multi-threading capability of ffmpeg.
|
|
41
|
+
const mediaDurations = await Promise.all(context.studio.beats.map(async (studioBeat, index) => {
|
|
12
42
|
const beat = context.studio.script.beats[index];
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
43
|
+
const movieDuration = await getMovieDulation(beat);
|
|
44
|
+
const audioDuration = studioBeat.audioFile ? await ffmpegGetMediaDuration(studioBeat.audioFile) : 0;
|
|
45
|
+
return {
|
|
46
|
+
movieDuration,
|
|
47
|
+
audioDuration,
|
|
48
|
+
};
|
|
49
|
+
}));
|
|
50
|
+
const inputIds = [];
|
|
51
|
+
const beatDurations = [];
|
|
52
|
+
context.studio.beats.reduce((spillover, studioBeat, index) => {
|
|
53
|
+
const beat = context.studio.script.beats[index];
|
|
54
|
+
const { audioDuration, movieDuration } = mediaDurations[index];
|
|
55
|
+
const paddingId = `[padding_${index}]`;
|
|
56
|
+
const canSpillover = index < context.studio.beats.length - 1 && mediaDurations[index + 1].movieDuration + mediaDurations[index + 1].audioDuration === 0;
|
|
21
57
|
if (studioBeat.audioFile) {
|
|
22
58
|
const audioId = FfmpegContextInputFormattedAudio(ffmpegContext, studioBeat.audioFile);
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
return 0;
|
|
29
|
-
}
|
|
30
|
-
return isClosingGap ? context.studio.script.audioParams.closingPadding : context.studio.script.audioParams.padding;
|
|
31
|
-
})();
|
|
32
|
-
const audioDuration = await ffmpegGetMediaDuration(studioBeat.audioFile);
|
|
33
|
-
const totalPadding = await (async () => {
|
|
34
|
-
if (movieDuration > 0) {
|
|
35
|
-
return padding + (movieDuration - audioDuration);
|
|
36
|
-
}
|
|
37
|
-
else if (beat.duration && beat.duration > audioDuration) {
|
|
38
|
-
return padding + (beat.duration - audioDuration);
|
|
39
|
-
}
|
|
40
|
-
return padding;
|
|
41
|
-
})();
|
|
42
|
-
studioBeat.duration = audioDuration + totalPadding;
|
|
59
|
+
// padding is the amount of audio padding specified in the script.
|
|
60
|
+
const padding = getPadding(context, beat, index);
|
|
61
|
+
// totalPadding is the amount of audio padding to be added to the audio file.
|
|
62
|
+
const totalPadding = getTotalPadding(padding, movieDuration, audioDuration, beat.duration, canSpillover);
|
|
63
|
+
beatDurations.push(audioDuration + totalPadding);
|
|
43
64
|
if (totalPadding > 0) {
|
|
44
65
|
const silentId = silentIds.pop();
|
|
45
|
-
ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${totalPadding}
|
|
46
|
-
|
|
66
|
+
ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${totalPadding}${paddingId}`);
|
|
67
|
+
inputIds.push(audioId, paddingId);
|
|
47
68
|
}
|
|
48
69
|
else {
|
|
49
|
-
|
|
70
|
+
inputIds.push(audioId);
|
|
71
|
+
if (totalPadding < 0) {
|
|
72
|
+
return -totalPadding;
|
|
73
|
+
}
|
|
50
74
|
}
|
|
51
75
|
}
|
|
52
76
|
else {
|
|
53
77
|
// NOTE: We come here when the text is empty and no audio property is specified.
|
|
54
|
-
|
|
78
|
+
const beatDuration = (() => {
|
|
79
|
+
const duration = beat.duration ?? (movieDuration > 0 ? movieDuration : 1.0);
|
|
80
|
+
if (!canSpillover && duration < spillover) {
|
|
81
|
+
return spillover; // We need to consume the spillover here.
|
|
82
|
+
}
|
|
83
|
+
return duration;
|
|
84
|
+
})();
|
|
85
|
+
beatDurations.push(beatDuration);
|
|
86
|
+
if (beatDuration <= spillover) {
|
|
87
|
+
return spillover - beatDuration;
|
|
88
|
+
}
|
|
55
89
|
const silentId = silentIds.pop();
|
|
56
|
-
ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${
|
|
57
|
-
|
|
90
|
+
ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${beatDuration - spillover}${paddingId}`);
|
|
91
|
+
inputIds.push(paddingId);
|
|
58
92
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
93
|
+
return 0;
|
|
94
|
+
}, 0);
|
|
95
|
+
assert(beatDurations.length === context.studio.beats.length, "beatDurations.length !== studio.beats.length");
|
|
96
|
+
// We need to "consume" extra silentIds.
|
|
97
|
+
silentIds.forEach((silentId, index) => {
|
|
98
|
+
const extraId = `[silent_extra_${index}]`;
|
|
99
|
+
ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${0.01}${extraId}`);
|
|
100
|
+
inputIds.push(extraId);
|
|
64
101
|
});
|
|
102
|
+
// Finally, combine all audio files.
|
|
65
103
|
ffmpegContext.filterComplex.push(`${inputIds.join("")}concat=n=${inputIds.length}:v=0:a=1[aout]`);
|
|
66
104
|
await FfmpegContextGenerateOutput(ffmpegContext, combinedFileName, ["-map", "[aout]"]);
|
|
105
|
+
const result = {
|
|
106
|
+
studio: {
|
|
107
|
+
...context.studio,
|
|
108
|
+
beats: context.studio.beats.map((studioBeat, index) => ({ ...studioBeat, duration: beatDurations[index] })),
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
// context.studio = result.studio; // TODO: removing this breaks test/test_movie.ts
|
|
67
112
|
return {
|
|
68
|
-
|
|
113
|
+
...context,
|
|
114
|
+
...result,
|
|
69
115
|
};
|
|
70
116
|
};
|
|
71
117
|
const combineAudioFilesAgentInfo = {
|
package/lib/agents/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
|
|
|
2
2
|
import combineAudioFilesAgent from "./combine_audio_files_agent.js";
|
|
3
3
|
import imageGoogleAgent from "./image_google_agent.js";
|
|
4
4
|
import imageOpenaiAgent from "./image_openai_agent.js";
|
|
5
|
+
import tavilySearchAgent from "./tavily_agent.js";
|
|
5
6
|
import movieGoogleAgent from "./movie_google_agent.js";
|
|
6
7
|
import mediaMockAgent from "./media_mock_agent.js";
|
|
7
8
|
import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
|
|
@@ -12,4 +13,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
|
|
|
12
13
|
import { textInputAgent } from "@graphai/input_agents";
|
|
13
14
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
14
15
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
15
|
-
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
|
|
16
|
+
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
|
package/lib/agents/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
|
|
|
2
2
|
import combineAudioFilesAgent from "./combine_audio_files_agent.js";
|
|
3
3
|
import imageGoogleAgent from "./image_google_agent.js";
|
|
4
4
|
import imageOpenaiAgent from "./image_openai_agent.js";
|
|
5
|
+
import tavilySearchAgent from "./tavily_agent.js";
|
|
5
6
|
import movieGoogleAgent from "./movie_google_agent.js";
|
|
6
7
|
import mediaMockAgent from "./media_mock_agent.js";
|
|
7
8
|
import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
|
|
@@ -13,4 +14,4 @@ import { textInputAgent } from "@graphai/input_agents";
|
|
|
13
14
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
14
15
|
// import * as vanilla from "@graphai/vanilla";
|
|
15
16
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
16
|
-
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
|
|
17
|
+
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { AgentFunction, AgentFunctionInfo, DefaultConfigData } from "graphai";
|
|
2
|
+
import { type TavilySearchResponse } from "@tavily/core";
|
|
3
|
+
type TavilySearchInputs = {
|
|
4
|
+
query: string;
|
|
5
|
+
};
|
|
6
|
+
type TavilySearchParams = {
|
|
7
|
+
apiKey?: string;
|
|
8
|
+
max_results?: number;
|
|
9
|
+
search_depth?: "basic" | "advanced";
|
|
10
|
+
include_answer?: boolean;
|
|
11
|
+
include_raw_content?: boolean | "markdown" | "text";
|
|
12
|
+
};
|
|
13
|
+
export declare const tavilySearchAgent: AgentFunction<TavilySearchParams, TavilySearchResponse, TavilySearchInputs, DefaultConfigData>;
|
|
14
|
+
declare const tavilySearchAgentInfo: AgentFunctionInfo;
|
|
15
|
+
export default tavilySearchAgentInfo;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { assert } from "graphai";
|
|
2
|
+
import { tavily } from "@tavily/core";
|
|
3
|
+
const getTavilyApiKey = (params, config) => {
|
|
4
|
+
if (params?.apiKey) {
|
|
5
|
+
return params.apiKey;
|
|
6
|
+
}
|
|
7
|
+
if (config?.apiKey) {
|
|
8
|
+
return config.apiKey;
|
|
9
|
+
}
|
|
10
|
+
return typeof process !== "undefined" ? process?.env?.TAVILY_API_KEY : null;
|
|
11
|
+
};
|
|
12
|
+
export const tavilySearchAgent = async ({ namedInputs, params, config, }) => {
|
|
13
|
+
const { query } = namedInputs;
|
|
14
|
+
assert(!!query, "tavilySearchAgent: query is required! set inputs: { query: 'search terms' }");
|
|
15
|
+
try {
|
|
16
|
+
const apiKey = getTavilyApiKey(params, config);
|
|
17
|
+
assert(apiKey, "Tavily API key is required. Please set the TAVILY_API_KEY environment variable or provide it in params/config.");
|
|
18
|
+
const tvly = tavily({ apiKey });
|
|
19
|
+
// Convert params to SDK options format
|
|
20
|
+
const sdkOptions = {};
|
|
21
|
+
if (params?.max_results !== undefined)
|
|
22
|
+
sdkOptions.maxResults = params.max_results;
|
|
23
|
+
if (params?.search_depth !== undefined)
|
|
24
|
+
sdkOptions.searchDepth = params.search_depth;
|
|
25
|
+
if (params?.include_answer !== undefined)
|
|
26
|
+
sdkOptions.includeAnswer = params.include_answer;
|
|
27
|
+
if (params?.include_raw_content !== undefined)
|
|
28
|
+
sdkOptions.includeRawContent = params.include_raw_content;
|
|
29
|
+
const response = await tvly.search(query, sdkOptions);
|
|
30
|
+
return response;
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
throw new Error(`Tavily search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
const tavilySearchAgentInfo = {
|
|
37
|
+
name: "tavilySearchAgent",
|
|
38
|
+
agent: tavilySearchAgent,
|
|
39
|
+
mock: tavilySearchAgent,
|
|
40
|
+
params: {
|
|
41
|
+
type: "object",
|
|
42
|
+
properties: {
|
|
43
|
+
apiKey: {
|
|
44
|
+
type: "string",
|
|
45
|
+
description: "Tavily API key",
|
|
46
|
+
},
|
|
47
|
+
max_results: {
|
|
48
|
+
type: "number",
|
|
49
|
+
description: "Maximum number of search results to return (default: 5)",
|
|
50
|
+
},
|
|
51
|
+
search_depth: {
|
|
52
|
+
type: "string",
|
|
53
|
+
enum: ["basic", "advanced"],
|
|
54
|
+
description: "Search depth - basic for faster results, advanced for more comprehensive results",
|
|
55
|
+
},
|
|
56
|
+
include_answer: {
|
|
57
|
+
type: "boolean",
|
|
58
|
+
description: "Include a direct answer to the query when available",
|
|
59
|
+
},
|
|
60
|
+
include_raw_content: {
|
|
61
|
+
type: "string",
|
|
62
|
+
enum: ["boolean", "markdown", "text"],
|
|
63
|
+
description: "Include raw content from search results (boolean, markdown, text)",
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
inputs: {
|
|
68
|
+
type: "object",
|
|
69
|
+
properties: {
|
|
70
|
+
query: {
|
|
71
|
+
type: "string",
|
|
72
|
+
description: "Search query string",
|
|
73
|
+
},
|
|
74
|
+
},
|
|
75
|
+
required: ["query"],
|
|
76
|
+
},
|
|
77
|
+
output: {
|
|
78
|
+
type: "object",
|
|
79
|
+
properties: {
|
|
80
|
+
query: { type: "string" },
|
|
81
|
+
answer: { type: ["string", "null"] },
|
|
82
|
+
results: {
|
|
83
|
+
type: "array",
|
|
84
|
+
items: {
|
|
85
|
+
type: "object",
|
|
86
|
+
properties: {
|
|
87
|
+
title: { type: "string" },
|
|
88
|
+
url: { type: "string" },
|
|
89
|
+
content: { type: "string" },
|
|
90
|
+
rawContent: { type: ["string", "null"] },
|
|
91
|
+
score: { type: "number" },
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
},
|
|
97
|
+
samples: [
|
|
98
|
+
{
|
|
99
|
+
inputs: {
|
|
100
|
+
query: "latest AI developments 2024",
|
|
101
|
+
},
|
|
102
|
+
params: {
|
|
103
|
+
max_results: 3,
|
|
104
|
+
include_answer: true,
|
|
105
|
+
},
|
|
106
|
+
result: {
|
|
107
|
+
query: "latest AI developments 2024",
|
|
108
|
+
answer: "Recent AI developments in 2024 include...",
|
|
109
|
+
results: [
|
|
110
|
+
{
|
|
111
|
+
title: "Major AI Breakthroughs in 2024",
|
|
112
|
+
url: "https://example.com/ai-2024",
|
|
113
|
+
content: "The year 2024 has seen significant advances in artificial intelligence...",
|
|
114
|
+
rawContent: null,
|
|
115
|
+
score: 0.95,
|
|
116
|
+
},
|
|
117
|
+
],
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
],
|
|
121
|
+
description: "Performs web search using Tavily API and returns relevant search results with optional AI-generated answers",
|
|
122
|
+
category: ["search", "web"],
|
|
123
|
+
author: "Receptron Team",
|
|
124
|
+
repository: "https://github.com/receptron/mulmocast-cli/tree/main/src/agents/tavily_agent.ts",
|
|
125
|
+
source: "https://github.com/receptron/mulmocast-cli/tree/main/src/agents/tavily_agent.ts",
|
|
126
|
+
package: "@receptron/mulmocast-cli",
|
|
127
|
+
license: "MIT",
|
|
128
|
+
environmentVariables: ["TAVILY_API_KEY"],
|
|
129
|
+
};
|
|
130
|
+
export default tavilySearchAgentInfo;
|
|
@@ -6,10 +6,5 @@ export const handler = async (argv) => {
|
|
|
6
6
|
process.exit(1);
|
|
7
7
|
}
|
|
8
8
|
await runTranslateIfNeeded(context, argv);
|
|
9
|
-
await audio(context);
|
|
10
|
-
await images(context);
|
|
11
|
-
if (context.caption) {
|
|
12
|
-
await captions(context);
|
|
13
|
-
}
|
|
14
|
-
await movie(context);
|
|
9
|
+
await audio(context).then(images).then(captions).then(movie);
|
|
15
10
|
};
|
package/lib/cli/common.d.ts
CHANGED
package/lib/cli/common.js
CHANGED
|
@@ -30,6 +30,12 @@ export const commonOptions = (yargs) => {
|
|
|
30
30
|
describe: "Dry run",
|
|
31
31
|
type: "boolean",
|
|
32
32
|
default: false,
|
|
33
|
+
})
|
|
34
|
+
.option("p", {
|
|
35
|
+
alias: "presentationStyle",
|
|
36
|
+
describe: "Presentation Style",
|
|
37
|
+
demandOption: false,
|
|
38
|
+
type: "string",
|
|
33
39
|
})
|
|
34
40
|
.positional("file", {
|
|
35
41
|
describe: "Mulmo Script File",
|