mulmocast 1.1.5 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/actions/audio.js +10 -1
- package/lib/actions/image_agents.d.ts +3 -12
- package/lib/actions/image_agents.js +12 -8
- package/lib/actions/images.js +2 -1
- package/lib/actions/translate.js +13 -31
- package/lib/agents/lipsync_replicate_agent.js +10 -3
- package/lib/cli/commands/audio/handler.js +1 -1
- package/lib/cli/commands/image/handler.js +1 -1
- package/lib/cli/commands/movie/handler.js +1 -1
- package/lib/cli/commands/pdf/handler.js +1 -1
- package/lib/cli/helpers.d.ts +1 -4
- package/lib/cli/helpers.js +3 -2
- package/lib/mcp/server.js +1 -1
- package/lib/methods/mulmo_presentation_style.d.ts +3 -2
- package/lib/methods/mulmo_script.js +4 -1
- package/lib/methods/mulmo_studio_context.d.ts +1 -0
- package/lib/methods/mulmo_studio_context.js +8 -0
- package/lib/types/agent.d.ts +1 -0
- package/lib/types/schema.d.ts +8 -8
- package/lib/types/schema.js +1 -1
- package/lib/types/type.d.ts +1 -1
- package/lib/utils/const.js +1 -1
- package/lib/utils/context.d.ts +376 -34
- package/lib/utils/context.js +95 -56
- package/lib/utils/filters.d.ts +1 -0
- package/lib/utils/filters.js +8 -0
- package/lib/utils/preprocess.d.ts +2 -2
- package/lib/utils/preprocess.js +3 -3
- package/lib/utils/provider2agent.d.ts +3 -2
- package/lib/utils/provider2agent.js +20 -2
- package/lib/utils/string.d.ts +1 -1
- package/lib/utils/string.js +11 -8
- package/package.json +1 -1
- package/scripts/templates/image_refs.json +1 -0
- package/scripts/templates/voice_over.json +1 -0
- package/scripts/test/gpt.json +1 -0
- package/scripts/test/test1.json +1 -0
- package/scripts/test/test_audio.json +1 -0
- package/scripts/test/test_audio_instructions.json +1 -0
- package/scripts/test/test_beats.json +1 -0
- package/scripts/test/test_captions.json +1 -0
- package/scripts/test/test_elevenlabs_models.json +1 -0
- package/scripts/test/test_hello.json +1 -0
- package/scripts/test/test_hello_google.json +1 -0
- package/scripts/test/test_html.json +1 -0
- package/scripts/test/test_image_refs.json +1 -0
- package/scripts/test/test_images.json +1 -0
- package/scripts/test/test_lang.json +58 -2
- package/scripts/test/test_layout.json +1 -0
- package/scripts/test/test_lipsync.json +9 -0
- package/scripts/test/test_loop.json +1 -0
- package/scripts/test/test_media.json +1 -0
- package/scripts/test/test_mixed_providers.json +1 -0
- package/scripts/test/test_movie.json +1 -0
- package/scripts/test/test_no_audio.json +1 -0
- package/scripts/test/test_no_audio_with_credit.json +1 -0
- package/scripts/test/test_order.json +1 -0
- package/scripts/test/test_order_portrait.json +1 -0
- package/scripts/test/test_replicate.json +19 -0
- package/scripts/test/test_slideout_left_no_audio.json +1 -0
- package/scripts/test/test_spillover.json +1 -0
- package/scripts/test/test_transition.json +1 -0
- package/scripts/test/test_transition_no_audio.json +1 -0
- package/scripts/test/test_video_speed.json +1 -0
- package/scripts/test/test_voice_over.json +1 -0
- package/scripts/test/test_voices.json +1 -0
- package/scripts/templates/image_prompt_only_template.ts +0 -95
package/lib/actions/audio.js
CHANGED
|
@@ -10,7 +10,7 @@ import ttsElevenlabsAgent from "../agents/tts_elevenlabs_agent.js";
|
|
|
10
10
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
11
11
|
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
12
12
|
import { text2SpeechProviderSchema } from "../types/index.js";
|
|
13
|
-
import { fileCacheAgentFilter } from "../utils/filters.js";
|
|
13
|
+
import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
|
|
14
14
|
import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
|
|
15
15
|
import { text2hash, localizedText, settings2GraphAIConfig } from "../utils/utils.js";
|
|
16
16
|
import { provider2TTSAgent } from "../utils/provider2agent.js";
|
|
@@ -58,6 +58,8 @@ const preprocessor = (namedInputs) => {
|
|
|
58
58
|
voiceId,
|
|
59
59
|
speechOptions,
|
|
60
60
|
model,
|
|
61
|
+
provider,
|
|
62
|
+
lang,
|
|
61
63
|
audioPath,
|
|
62
64
|
studioBeat,
|
|
63
65
|
needsTTS,
|
|
@@ -84,6 +86,8 @@ const graph_tts = {
|
|
|
84
86
|
agent: ":preprocessor.ttsAgent",
|
|
85
87
|
inputs: {
|
|
86
88
|
text: ":preprocessor.text",
|
|
89
|
+
provider: ":preprocessor.provider",
|
|
90
|
+
lang: ":preprocessor.lang",
|
|
87
91
|
cache: {
|
|
88
92
|
force: [":context.force"],
|
|
89
93
|
file: ":preprocessor.audioPath",
|
|
@@ -173,6 +177,11 @@ const agentFilters = [
|
|
|
173
177
|
agent: fileCacheAgentFilter,
|
|
174
178
|
nodeIds: ["tts"],
|
|
175
179
|
},
|
|
180
|
+
{
|
|
181
|
+
name: "nijovoiceTextAgentFilter",
|
|
182
|
+
agent: nijovoiceTextAgentFilter,
|
|
183
|
+
nodeIds: ["tts"],
|
|
184
|
+
},
|
|
176
185
|
];
|
|
177
186
|
const getConcurrency = (context) => {
|
|
178
187
|
// Check if any speaker uses nijivoice or elevenlabs (providers that require concurrency = 1)
|
|
@@ -23,10 +23,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
23
23
|
};
|
|
24
24
|
lipSyncFile?: string;
|
|
25
25
|
lipSyncModel?: string;
|
|
26
|
-
|
|
27
|
-
agentName: string;
|
|
28
|
-
defaultModel: string;
|
|
29
|
-
};
|
|
26
|
+
lipSyncAgentName?: string;
|
|
30
27
|
audioFile?: string;
|
|
31
28
|
beatDuration?: number;
|
|
32
29
|
htmlPrompt?: undefined;
|
|
@@ -61,10 +58,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
61
58
|
};
|
|
62
59
|
lipSyncFile?: string;
|
|
63
60
|
lipSyncModel?: string;
|
|
64
|
-
|
|
65
|
-
agentName: string;
|
|
66
|
-
defaultModel: string;
|
|
67
|
-
};
|
|
61
|
+
lipSyncAgentName?: string;
|
|
68
62
|
audioFile?: string;
|
|
69
63
|
beatDuration?: number;
|
|
70
64
|
htmlPrompt?: undefined;
|
|
@@ -102,10 +96,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
|
|
|
102
96
|
};
|
|
103
97
|
lipSyncFile?: string;
|
|
104
98
|
lipSyncModel?: string;
|
|
105
|
-
|
|
106
|
-
agentName: string;
|
|
107
|
-
defaultModel: string;
|
|
108
|
-
};
|
|
99
|
+
lipSyncAgentName?: string;
|
|
109
100
|
audioFile?: string;
|
|
110
101
|
beatDuration?: number;
|
|
111
102
|
htmlPrompt?: undefined;
|
|
@@ -25,16 +25,20 @@ export const imagePreprocessAgent = async (namedInputs) => {
|
|
|
25
25
|
movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
|
|
26
26
|
beatDuration: beat.duration ?? studioBeat?.duration,
|
|
27
27
|
};
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
28
|
+
const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
|
|
29
|
+
if (isMovie) {
|
|
30
|
+
if (beat.soundEffectPrompt) {
|
|
31
|
+
returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
|
|
32
|
+
returnValue.soundEffectModel =
|
|
33
|
+
beat.soundEffectParams?.model ?? context.presentationStyle.soundEffectParams?.model ?? returnValue.soundEffectAgentInfo.defaultModel;
|
|
34
|
+
returnValue.soundEffectFile = moviePaths.soundEffectFile;
|
|
35
|
+
returnValue.soundEffectPrompt = beat.soundEffectPrompt;
|
|
36
|
+
}
|
|
34
37
|
}
|
|
35
38
|
if (beat.enableLipSync) {
|
|
36
|
-
|
|
37
|
-
returnValue.
|
|
39
|
+
const lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
|
|
40
|
+
returnValue.lipSyncAgentName = lipSyncAgentInfo.agentName;
|
|
41
|
+
returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? lipSyncAgentInfo.defaultModel;
|
|
38
42
|
returnValue.lipSyncFile = moviePaths.lipSyncFile;
|
|
39
43
|
// Audio file will be set from the beat's audio file when available
|
|
40
44
|
returnValue.audioFile = studioBeat?.audioFile;
|
package/lib/actions/images.js
CHANGED
|
@@ -218,10 +218,11 @@ const beat_graph_data = {
|
|
|
218
218
|
},
|
|
219
219
|
lipSyncGenerator: {
|
|
220
220
|
if: ":beat.enableLipSync",
|
|
221
|
-
agent: ":preprocessor.
|
|
221
|
+
agent: ":preprocessor.lipSyncAgentName",
|
|
222
222
|
inputs: {
|
|
223
223
|
onComplete: [":soundEffectGenerator"], // to wait for soundEffectGenerator to finish
|
|
224
224
|
movieFile: ":preprocessor.movieFile",
|
|
225
|
+
imageFile: ":preprocessor.referenceImageForMovie",
|
|
225
226
|
audioFile: ":preprocessor.audioFile",
|
|
226
227
|
lipSyncFile: ":preprocessor.lipSyncFile",
|
|
227
228
|
params: {
|
package/lib/actions/translate.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
|
-
import { GraphAI, assert } from "graphai";
|
|
2
|
+
import { GraphAI, assert, isNull } from "graphai";
|
|
3
3
|
import * as agents from "@graphai/vanilla";
|
|
4
4
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
5
5
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
6
|
-
import { recursiveSplitJa
|
|
6
|
+
import { recursiveSplitJa } from "../utils/string.js";
|
|
7
7
|
import { settings2GraphAIConfig } from "../utils/utils.js";
|
|
8
8
|
import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
|
|
9
9
|
import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
|
|
@@ -13,17 +13,9 @@ const translateGraph = {
|
|
|
13
13
|
version: 0.5,
|
|
14
14
|
nodes: {
|
|
15
15
|
context: {},
|
|
16
|
-
defaultLang: {},
|
|
17
16
|
outDirPath: {},
|
|
18
17
|
outputMultilingualFilePath: {},
|
|
19
|
-
|
|
20
|
-
agent: "stringUpdateTextAgent",
|
|
21
|
-
inputs: {
|
|
22
|
-
newText: ":context.studio.script.lang",
|
|
23
|
-
oldText: ":defaultLang",
|
|
24
|
-
},
|
|
25
|
-
},
|
|
26
|
-
targetLangs: {}, // TODO
|
|
18
|
+
targetLangs: {},
|
|
27
19
|
mergeStudioResult: {
|
|
28
20
|
isResult: true,
|
|
29
21
|
agent: "mergeObjectAgent",
|
|
@@ -37,7 +29,6 @@ const translateGraph = {
|
|
|
37
29
|
targetLangs: ":targetLangs",
|
|
38
30
|
context: ":context",
|
|
39
31
|
rows: ":context.studio.script.beats",
|
|
40
|
-
lang: ":lang",
|
|
41
32
|
},
|
|
42
33
|
params: {
|
|
43
34
|
rowKey: "beat",
|
|
@@ -62,7 +53,7 @@ const translateGraph = {
|
|
|
62
53
|
beat: ":beat",
|
|
63
54
|
multiLingual: ":multiLingual",
|
|
64
55
|
rows: ":targetLangs",
|
|
65
|
-
lang: ":lang
|
|
56
|
+
lang: ":context.studio.script.lang",
|
|
66
57
|
context: ":context",
|
|
67
58
|
beatIndex: ":__mapIndex",
|
|
68
59
|
},
|
|
@@ -120,17 +111,11 @@ const translateGraph = {
|
|
|
120
111
|
},
|
|
121
112
|
ttsTexts: {
|
|
122
113
|
agent: (namedInputs) => {
|
|
123
|
-
const { localizedText
|
|
114
|
+
const { localizedText } = namedInputs;
|
|
124
115
|
// cache
|
|
125
116
|
if (localizedText.ttsTexts) {
|
|
126
117
|
return localizedText;
|
|
127
118
|
}
|
|
128
|
-
if (targetLang === "ja") {
|
|
129
|
-
return {
|
|
130
|
-
...localizedText,
|
|
131
|
-
ttsTexts: localizedText?.texts?.map((text) => replacePairsJa(text, replacementsJa)),
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
119
|
return {
|
|
135
120
|
...localizedText,
|
|
136
121
|
ttsTexts: localizedText.texts,
|
|
@@ -180,18 +165,14 @@ const localizedTextCacheAgentFilter = async (context, next) => {
|
|
|
180
165
|
if (!beat.text) {
|
|
181
166
|
return { text: "" };
|
|
182
167
|
}
|
|
183
|
-
// The original text is unchanged and the target language text is present
|
|
184
|
-
if (multiLingual.multiLingualTexts &&
|
|
185
|
-
multiLingual.multiLingualTexts[lang] &&
|
|
186
|
-
multiLingual.multiLingualTexts[lang].text === beat.text &&
|
|
187
|
-
multiLingual.multiLingualTexts[targetLang] &&
|
|
188
|
-
multiLingual.multiLingualTexts[targetLang].text) {
|
|
189
|
-
return { text: multiLingual.multiLingualTexts[targetLang].text };
|
|
190
|
-
}
|
|
191
168
|
// same language
|
|
192
169
|
if (targetLang === lang) {
|
|
193
170
|
return { text: beat.text };
|
|
194
171
|
}
|
|
172
|
+
// The original text is unchanged and the target language text is present
|
|
173
|
+
if (multiLingual.multiLingualTexts?.[lang]?.text === beat.text && multiLingual.multiLingualTexts[targetLang]?.text) {
|
|
174
|
+
return { text: multiLingual.multiLingualTexts[targetLang].text };
|
|
175
|
+
}
|
|
195
176
|
try {
|
|
196
177
|
MulmoStudioContextMethods.setBeatSessionState(mulmoContext, "multiLingual", beatIndex, true);
|
|
197
178
|
return await next(context);
|
|
@@ -207,8 +188,6 @@ const agentFilters = [
|
|
|
207
188
|
nodeIds: ["localizedTexts"],
|
|
208
189
|
},
|
|
209
190
|
];
|
|
210
|
-
const defaultLang = "en";
|
|
211
|
-
const targetLangs = ["ja", "en"];
|
|
212
191
|
export const translate = async (context, args) => {
|
|
213
192
|
const { settings, callbacks } = args ?? {};
|
|
214
193
|
try {
|
|
@@ -217,11 +196,14 @@ export const translate = async (context, args) => {
|
|
|
217
196
|
const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
|
|
218
197
|
const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
|
|
219
198
|
mkdir(outDirPath);
|
|
199
|
+
const langs = (context.multiLingual ?? []).map((x) => Object.keys(x.multiLingualTexts)).flat(); // existing langs in multiLingual
|
|
200
|
+
const targetLangs = [
|
|
201
|
+
...new Set([context.studio.script.lang, langs, context.lang, context.studio.script.captionParams?.lang].flat().filter((x) => !isNull(x))),
|
|
202
|
+
];
|
|
220
203
|
const config = settings2GraphAIConfig(settings, process.env);
|
|
221
204
|
assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
|
|
222
205
|
const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
|
|
223
206
|
graph.injectValue("context", context);
|
|
224
|
-
graph.injectValue("defaultLang", defaultLang);
|
|
225
207
|
graph.injectValue("targetLangs", targetLangs);
|
|
226
208
|
graph.injectValue("outDirPath", outDirPath);
|
|
227
209
|
graph.injectValue("outputMultilingualFilePath", outputMultilingualFilePath);
|
|
@@ -3,7 +3,7 @@ import { GraphAILogger } from "graphai";
|
|
|
3
3
|
import Replicate from "replicate";
|
|
4
4
|
import { provider2LipSyncAgent } from "../utils/provider2agent.js";
|
|
5
5
|
export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
6
|
-
const { movieFile, audioFile } = namedInputs;
|
|
6
|
+
const { movieFile, audioFile, imageFile } = namedInputs;
|
|
7
7
|
const apiKey = config?.apiKey;
|
|
8
8
|
const model = params.model ?? provider2LipSyncAgent.replicate.defaultModel;
|
|
9
9
|
if (!apiKey) {
|
|
@@ -12,10 +12,12 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
|
|
|
12
12
|
const replicate = new Replicate({
|
|
13
13
|
auth: apiKey,
|
|
14
14
|
});
|
|
15
|
-
const videoBuffer = readFileSync(movieFile);
|
|
15
|
+
const videoBuffer = movieFile ? readFileSync(movieFile) : undefined;
|
|
16
16
|
const audioBuffer = readFileSync(audioFile);
|
|
17
|
-
const
|
|
17
|
+
const imageBuffer = imageFile ? readFileSync(imageFile) : undefined;
|
|
18
|
+
const videoUri = videoBuffer ? `data:video/quicktime;base64,${videoBuffer.toString("base64")}` : undefined;
|
|
18
19
|
const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
|
|
20
|
+
const imageUri = imageBuffer ? `data:image/png;base64,${imageBuffer.toString("base64")}` : undefined;
|
|
19
21
|
const input = {
|
|
20
22
|
video: undefined,
|
|
21
23
|
video_input: undefined,
|
|
@@ -23,6 +25,7 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
|
|
|
23
25
|
audio: undefined,
|
|
24
26
|
audio_input: undefined,
|
|
25
27
|
audio_file: undefined,
|
|
28
|
+
image: undefined,
|
|
26
29
|
};
|
|
27
30
|
const modelParams = provider2LipSyncAgent.replicate.modelParams[model];
|
|
28
31
|
if (!modelParams) {
|
|
@@ -30,12 +33,16 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
|
|
|
30
33
|
}
|
|
31
34
|
const videoParam = modelParams.video;
|
|
32
35
|
const audioParam = modelParams.audio;
|
|
36
|
+
const imageParam = modelParams.image;
|
|
33
37
|
if (videoParam === "video" || videoParam === "video_input" || videoParam === "video_url") {
|
|
34
38
|
input[videoParam] = videoUri;
|
|
35
39
|
}
|
|
36
40
|
if (audioParam === "audio" || audioParam === "audio_input" || audioParam === "audio_file") {
|
|
37
41
|
input[audioParam] = audioUri;
|
|
38
42
|
}
|
|
43
|
+
if (imageParam === "image") {
|
|
44
|
+
input[imageParam] = imageUri;
|
|
45
|
+
}
|
|
39
46
|
const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
|
|
40
47
|
try {
|
|
41
48
|
const output = await replicate.run(model_identifier, {
|
package/lib/cli/helpers.d.ts
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
import type { CliArgs } from "../types/cli_types.js";
|
|
2
2
|
import { FileObject, InitOptions, MulmoStudioContext } from "../types/index.js";
|
|
3
|
-
export declare const runTranslateIfNeeded: (context: MulmoStudioContext,
|
|
4
|
-
l?: string;
|
|
5
|
-
c?: string;
|
|
6
|
-
}) => Promise<void>;
|
|
3
|
+
export declare const runTranslateIfNeeded: (context: MulmoStudioContext, includeCaption?: boolean) => Promise<void>;
|
|
7
4
|
export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
|
|
8
5
|
export declare const getFileObject: (args: {
|
|
9
6
|
basedir?: string;
|
package/lib/cli/helpers.js
CHANGED
|
@@ -5,10 +5,11 @@ import clipboardy from "clipboardy";
|
|
|
5
5
|
import { getBaseDirPath, getFullPath, getOutputStudioFilePath, resolveDirPath, mkdir, getOutputMultilingualFilePath, generateTimestampedFileName, } from "../utils/file.js";
|
|
6
6
|
import { isHttp } from "../utils/utils.js";
|
|
7
7
|
import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
|
|
8
|
+
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
8
9
|
import { translate } from "../actions/translate.js";
|
|
9
10
|
import { initializeContextFromFiles } from "../utils/context.js";
|
|
10
|
-
export const runTranslateIfNeeded = async (context,
|
|
11
|
-
if (
|
|
11
|
+
export const runTranslateIfNeeded = async (context, includeCaption = false) => {
|
|
12
|
+
if (MulmoStudioContextMethods.needTranslate(context, includeCaption)) {
|
|
12
13
|
GraphAILogger.log("run translate");
|
|
13
14
|
await translate(context);
|
|
14
15
|
}
|
package/lib/mcp/server.js
CHANGED
|
@@ -104,7 +104,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
104
104
|
throw new Error("Failed to initialize context from MulmoScript");
|
|
105
105
|
}
|
|
106
106
|
// Run translation if needed
|
|
107
|
-
await runTranslateIfNeeded(context
|
|
107
|
+
await runTranslateIfNeeded(context);
|
|
108
108
|
// Execute the requested command
|
|
109
109
|
switch (cmd) {
|
|
110
110
|
case "movie":
|
|
@@ -36,9 +36,10 @@ export declare const MulmoPresentationStyleMethods: {
|
|
|
36
36
|
defaultModel: import("../utils/provider2agent.js").ReplicateModel;
|
|
37
37
|
models: import("../utils/provider2agent.js").ReplicateModel[];
|
|
38
38
|
modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
|
|
39
|
-
identifier?: `${string}/${string}:${string}`;
|
|
40
|
-
video
|
|
39
|
+
identifier?: `${string}/${string}:${string}` | `${string}/${string}`;
|
|
40
|
+
video?: string;
|
|
41
41
|
audio: string;
|
|
42
|
+
image?: string;
|
|
42
43
|
}>;
|
|
43
44
|
};
|
|
44
45
|
getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
|
|
@@ -18,6 +18,9 @@ const validators = [{ from: "1.0", to: "1.1", validator: validate_1_0 }];
|
|
|
18
18
|
export const MulmoScriptMethods = {
|
|
19
19
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
20
20
|
validate(script) {
|
|
21
|
+
const version = script.$mulmocast.version;
|
|
22
|
+
// lang was optional in 1.0 and 1.1
|
|
23
|
+
const defaultLang = version === "1.0" || version === "1.1" ? { lang: "en" } : {};
|
|
21
24
|
const validatedScript = validators.reduce((acc, validator) => {
|
|
22
25
|
if (acc.$mulmocast.version === validator.from) {
|
|
23
26
|
const validated = validator.validator(acc);
|
|
@@ -25,7 +28,7 @@ export const MulmoScriptMethods = {
|
|
|
25
28
|
return validated;
|
|
26
29
|
}
|
|
27
30
|
return acc;
|
|
28
|
-
}, script);
|
|
31
|
+
}, { ...defaultLang, ...script });
|
|
29
32
|
return mulmoScriptSchema.parse(validatedScript);
|
|
30
33
|
},
|
|
31
34
|
};
|
|
@@ -11,4 +11,5 @@ export declare const MulmoStudioContextMethods: {
|
|
|
11
11
|
getCaption(context: MulmoStudioContext): string | undefined;
|
|
12
12
|
setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
|
|
13
13
|
setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
|
|
14
|
+
needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
|
|
14
15
|
};
|
|
@@ -63,4 +63,12 @@ export const MulmoStudioContextMethods = {
|
|
|
63
63
|
}
|
|
64
64
|
notifyBeatStateChange(context, sessionType, index);
|
|
65
65
|
},
|
|
66
|
+
needTranslate(context, includeCaption = false) {
|
|
67
|
+
// context.studio.script.lang = defaultLang, context.lang = targetLanguage.
|
|
68
|
+
if (includeCaption) {
|
|
69
|
+
return (context.studio.script.lang !== context.lang ||
|
|
70
|
+
(context.studio.script.captionParams?.lang && context.studio.script.lang !== context.studio.script.captionParams?.lang));
|
|
71
|
+
}
|
|
72
|
+
return context.studio.script.lang !== context.lang;
|
|
73
|
+
},
|
|
66
74
|
};
|
package/lib/types/agent.d.ts
CHANGED
package/lib/types/schema.d.ts
CHANGED
|
@@ -3861,7 +3861,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
|
|
|
3861
3861
|
title?: string | undefined;
|
|
3862
3862
|
description?: string | undefined;
|
|
3863
3863
|
}>, "many">>;
|
|
3864
|
-
lang: z.
|
|
3864
|
+
lang: z.ZodString;
|
|
3865
3865
|
beats: z.ZodArray<z.ZodObject<{
|
|
3866
3866
|
speaker: z.ZodOptional<z.ZodString>;
|
|
3867
3867
|
text: z.ZodDefault<z.ZodString>;
|
|
@@ -5036,6 +5036,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
|
|
|
5036
5036
|
imagePath: z.ZodOptional<z.ZodString>;
|
|
5037
5037
|
__test_invalid__: z.ZodOptional<z.ZodBoolean>;
|
|
5038
5038
|
}, "strict", z.ZodTypeAny, {
|
|
5039
|
+
lang: string;
|
|
5039
5040
|
imageParams: {
|
|
5040
5041
|
provider: string;
|
|
5041
5042
|
model?: string | undefined;
|
|
@@ -5332,7 +5333,6 @@ export declare const mulmoScriptSchema: z.ZodObject<{
|
|
|
5332
5333
|
} | undefined;
|
|
5333
5334
|
enableLipSync?: boolean | undefined;
|
|
5334
5335
|
}[];
|
|
5335
|
-
lang?: string | undefined;
|
|
5336
5336
|
title?: string | undefined;
|
|
5337
5337
|
description?: string | undefined;
|
|
5338
5338
|
lipSyncParams?: {
|
|
@@ -5359,6 +5359,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
|
|
|
5359
5359
|
imagePath?: string | undefined;
|
|
5360
5360
|
__test_invalid__?: boolean | undefined;
|
|
5361
5361
|
}, {
|
|
5362
|
+
lang: string;
|
|
5362
5363
|
$mulmocast: {
|
|
5363
5364
|
version: "1.1";
|
|
5364
5365
|
credit?: "closing" | undefined;
|
|
@@ -5564,7 +5565,6 @@ export declare const mulmoScriptSchema: z.ZodObject<{
|
|
|
5564
5565
|
} | undefined;
|
|
5565
5566
|
enableLipSync?: boolean | undefined;
|
|
5566
5567
|
}[];
|
|
5567
|
-
lang?: string | undefined;
|
|
5568
5568
|
title?: string | undefined;
|
|
5569
5569
|
description?: string | undefined;
|
|
5570
5570
|
imageParams?: {
|
|
@@ -6393,7 +6393,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
6393
6393
|
title?: string | undefined;
|
|
6394
6394
|
description?: string | undefined;
|
|
6395
6395
|
}>, "many">>;
|
|
6396
|
-
lang: z.
|
|
6396
|
+
lang: z.ZodString;
|
|
6397
6397
|
beats: z.ZodArray<z.ZodObject<{
|
|
6398
6398
|
speaker: z.ZodOptional<z.ZodString>;
|
|
6399
6399
|
text: z.ZodDefault<z.ZodString>;
|
|
@@ -7568,6 +7568,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
7568
7568
|
imagePath: z.ZodOptional<z.ZodString>;
|
|
7569
7569
|
__test_invalid__: z.ZodOptional<z.ZodBoolean>;
|
|
7570
7570
|
}, "strict", z.ZodTypeAny, {
|
|
7571
|
+
lang: string;
|
|
7571
7572
|
imageParams: {
|
|
7572
7573
|
provider: string;
|
|
7573
7574
|
model?: string | undefined;
|
|
@@ -7864,7 +7865,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
7864
7865
|
} | undefined;
|
|
7865
7866
|
enableLipSync?: boolean | undefined;
|
|
7866
7867
|
}[];
|
|
7867
|
-
lang?: string | undefined;
|
|
7868
7868
|
title?: string | undefined;
|
|
7869
7869
|
description?: string | undefined;
|
|
7870
7870
|
lipSyncParams?: {
|
|
@@ -7891,6 +7891,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
7891
7891
|
imagePath?: string | undefined;
|
|
7892
7892
|
__test_invalid__?: boolean | undefined;
|
|
7893
7893
|
}, {
|
|
7894
|
+
lang: string;
|
|
7894
7895
|
$mulmocast: {
|
|
7895
7896
|
version: "1.1";
|
|
7896
7897
|
credit?: "closing" | undefined;
|
|
@@ -8096,7 +8097,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
8096
8097
|
} | undefined;
|
|
8097
8098
|
enableLipSync?: boolean | undefined;
|
|
8098
8099
|
}[];
|
|
8099
|
-
lang?: string | undefined;
|
|
8100
8100
|
title?: string | undefined;
|
|
8101
8101
|
description?: string | undefined;
|
|
8102
8102
|
imageParams?: {
|
|
@@ -8275,6 +8275,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
8275
8275
|
captionFile?: string | undefined;
|
|
8276
8276
|
}[];
|
|
8277
8277
|
script: {
|
|
8278
|
+
lang: string;
|
|
8278
8279
|
imageParams: {
|
|
8279
8280
|
provider: string;
|
|
8280
8281
|
model?: string | undefined;
|
|
@@ -8571,7 +8572,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
8571
8572
|
} | undefined;
|
|
8572
8573
|
enableLipSync?: boolean | undefined;
|
|
8573
8574
|
}[];
|
|
8574
|
-
lang?: string | undefined;
|
|
8575
8575
|
title?: string | undefined;
|
|
8576
8576
|
description?: string | undefined;
|
|
8577
8577
|
lipSyncParams?: {
|
|
@@ -8616,6 +8616,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
8616
8616
|
captionFile?: string | undefined;
|
|
8617
8617
|
}[];
|
|
8618
8618
|
script: {
|
|
8619
|
+
lang: string;
|
|
8619
8620
|
$mulmocast: {
|
|
8620
8621
|
version: "1.1";
|
|
8621
8622
|
credit?: "closing" | undefined;
|
|
@@ -8821,7 +8822,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
|
|
|
8821
8822
|
} | undefined;
|
|
8822
8823
|
enableLipSync?: boolean | undefined;
|
|
8823
8824
|
}[];
|
|
8824
|
-
lang?: string | undefined;
|
|
8825
8825
|
title?: string | undefined;
|
|
8826
8826
|
description?: string | undefined;
|
|
8827
8827
|
imageParams?: {
|
package/lib/types/schema.js
CHANGED
|
@@ -371,7 +371,7 @@ export const mulmoScriptSchema = mulmoPresentationStyleSchema
|
|
|
371
371
|
title: z.string().optional(),
|
|
372
372
|
description: z.string().optional(),
|
|
373
373
|
references: z.array(mulmoReferenceSchema).optional(),
|
|
374
|
-
lang: langSchema
|
|
374
|
+
lang: langSchema, // required (default WAS "en")
|
|
375
375
|
beats: z.array(mulmoBeatSchema).min(1),
|
|
376
376
|
// TODO: Delete it later
|
|
377
377
|
imagePath: z.string().optional(), // for keynote images movie ??
|
package/lib/types/type.d.ts
CHANGED
package/lib/utils/const.js
CHANGED
|
@@ -4,7 +4,7 @@ export const imageDirName = "images";
|
|
|
4
4
|
export const cacheDirName = "cache";
|
|
5
5
|
export const pdf_modes = ["slide", "talk", "handout"];
|
|
6
6
|
export const pdf_sizes = ["letter", "a4"];
|
|
7
|
-
export const languages = ["en", "ja"];
|
|
7
|
+
export const languages = ["en", "ja", "fr", "es"];
|
|
8
8
|
export const storyToScriptGenerateMode = {
|
|
9
9
|
stepWise: "step_wise",
|
|
10
10
|
oneStep: "one_step",
|