mulmocast 1.1.5 → 1.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/lib/actions/audio.js +10 -1
  2. package/lib/actions/image_agents.d.ts +3 -12
  3. package/lib/actions/image_agents.js +12 -8
  4. package/lib/actions/images.js +2 -1
  5. package/lib/actions/translate.js +13 -31
  6. package/lib/agents/lipsync_replicate_agent.js +10 -3
  7. package/lib/cli/commands/audio/handler.js +1 -1
  8. package/lib/cli/commands/image/handler.js +1 -1
  9. package/lib/cli/commands/movie/handler.js +1 -1
  10. package/lib/cli/commands/pdf/handler.js +1 -1
  11. package/lib/cli/helpers.d.ts +1 -4
  12. package/lib/cli/helpers.js +3 -2
  13. package/lib/mcp/server.js +1 -1
  14. package/lib/methods/mulmo_presentation_style.d.ts +3 -2
  15. package/lib/methods/mulmo_script.js +4 -1
  16. package/lib/methods/mulmo_studio_context.d.ts +1 -0
  17. package/lib/methods/mulmo_studio_context.js +8 -0
  18. package/lib/types/agent.d.ts +1 -0
  19. package/lib/types/schema.d.ts +8 -8
  20. package/lib/types/schema.js +1 -1
  21. package/lib/types/type.d.ts +1 -1
  22. package/lib/utils/const.js +1 -1
  23. package/lib/utils/context.d.ts +376 -34
  24. package/lib/utils/context.js +95 -56
  25. package/lib/utils/filters.d.ts +1 -0
  26. package/lib/utils/filters.js +8 -0
  27. package/lib/utils/preprocess.d.ts +2 -2
  28. package/lib/utils/preprocess.js +3 -3
  29. package/lib/utils/provider2agent.d.ts +3 -2
  30. package/lib/utils/provider2agent.js +20 -2
  31. package/lib/utils/string.d.ts +1 -1
  32. package/lib/utils/string.js +11 -8
  33. package/package.json +1 -1
  34. package/scripts/templates/image_refs.json +1 -0
  35. package/scripts/templates/voice_over.json +1 -0
  36. package/scripts/test/gpt.json +1 -0
  37. package/scripts/test/test1.json +1 -0
  38. package/scripts/test/test_audio.json +1 -0
  39. package/scripts/test/test_audio_instructions.json +1 -0
  40. package/scripts/test/test_beats.json +1 -0
  41. package/scripts/test/test_captions.json +1 -0
  42. package/scripts/test/test_elevenlabs_models.json +1 -0
  43. package/scripts/test/test_hello.json +1 -0
  44. package/scripts/test/test_hello_google.json +1 -0
  45. package/scripts/test/test_html.json +1 -0
  46. package/scripts/test/test_image_refs.json +1 -0
  47. package/scripts/test/test_images.json +1 -0
  48. package/scripts/test/test_lang.json +58 -2
  49. package/scripts/test/test_layout.json +1 -0
  50. package/scripts/test/test_lipsync.json +9 -0
  51. package/scripts/test/test_loop.json +1 -0
  52. package/scripts/test/test_media.json +1 -0
  53. package/scripts/test/test_mixed_providers.json +1 -0
  54. package/scripts/test/test_movie.json +1 -0
  55. package/scripts/test/test_no_audio.json +1 -0
  56. package/scripts/test/test_no_audio_with_credit.json +1 -0
  57. package/scripts/test/test_order.json +1 -0
  58. package/scripts/test/test_order_portrait.json +1 -0
  59. package/scripts/test/test_replicate.json +19 -0
  60. package/scripts/test/test_slideout_left_no_audio.json +1 -0
  61. package/scripts/test/test_spillover.json +1 -0
  62. package/scripts/test/test_transition.json +1 -0
  63. package/scripts/test/test_transition_no_audio.json +1 -0
  64. package/scripts/test/test_video_speed.json +1 -0
  65. package/scripts/test/test_voice_over.json +1 -0
  66. package/scripts/test/test_voices.json +1 -0
  67. package/scripts/templates/image_prompt_only_template.ts +0 -95
@@ -10,7 +10,7 @@ import ttsElevenlabsAgent from "../agents/tts_elevenlabs_agent.js";
10
10
  import { fileWriteAgent } from "@graphai/vanilla_node_agents";
11
11
  import { MulmoPresentationStyleMethods } from "../methods/index.js";
12
12
  import { text2SpeechProviderSchema } from "../types/index.js";
13
- import { fileCacheAgentFilter } from "../utils/filters.js";
13
+ import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
14
14
  import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
15
15
  import { text2hash, localizedText, settings2GraphAIConfig } from "../utils/utils.js";
16
16
  import { provider2TTSAgent } from "../utils/provider2agent.js";
@@ -58,6 +58,8 @@ const preprocessor = (namedInputs) => {
58
58
  voiceId,
59
59
  speechOptions,
60
60
  model,
61
+ provider,
62
+ lang,
61
63
  audioPath,
62
64
  studioBeat,
63
65
  needsTTS,
@@ -84,6 +86,8 @@ const graph_tts = {
84
86
  agent: ":preprocessor.ttsAgent",
85
87
  inputs: {
86
88
  text: ":preprocessor.text",
89
+ provider: ":preprocessor.provider",
90
+ lang: ":preprocessor.lang",
87
91
  cache: {
88
92
  force: [":context.force"],
89
93
  file: ":preprocessor.audioPath",
@@ -173,6 +177,11 @@ const agentFilters = [
173
177
  agent: fileCacheAgentFilter,
174
178
  nodeIds: ["tts"],
175
179
  },
180
+ {
181
+ name: "nijovoiceTextAgentFilter",
182
+ agent: nijovoiceTextAgentFilter,
183
+ nodeIds: ["tts"],
184
+ },
176
185
  ];
177
186
  const getConcurrency = (context) => {
178
187
  // Check if any speaker uses nijivoice or elevenlabs (providers that require concurrency = 1)
@@ -23,10 +23,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
23
23
  };
24
24
  lipSyncFile?: string;
25
25
  lipSyncModel?: string;
26
- lipSyncAgentInfo?: {
27
- agentName: string;
28
- defaultModel: string;
29
- };
26
+ lipSyncAgentName?: string;
30
27
  audioFile?: string;
31
28
  beatDuration?: number;
32
29
  htmlPrompt?: undefined;
@@ -61,10 +58,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
61
58
  };
62
59
  lipSyncFile?: string;
63
60
  lipSyncModel?: string;
64
- lipSyncAgentInfo?: {
65
- agentName: string;
66
- defaultModel: string;
67
- };
61
+ lipSyncAgentName?: string;
68
62
  audioFile?: string;
69
63
  beatDuration?: number;
70
64
  htmlPrompt?: undefined;
@@ -102,10 +96,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
102
96
  };
103
97
  lipSyncFile?: string;
104
98
  lipSyncModel?: string;
105
- lipSyncAgentInfo?: {
106
- agentName: string;
107
- defaultModel: string;
108
- };
99
+ lipSyncAgentName?: string;
109
100
  audioFile?: string;
110
101
  beatDuration?: number;
111
102
  htmlPrompt?: undefined;
@@ -25,16 +25,20 @@ export const imagePreprocessAgent = async (namedInputs) => {
25
25
  movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
26
26
  beatDuration: beat.duration ?? studioBeat?.duration,
27
27
  };
28
- if (beat.soundEffectPrompt) {
29
- returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
30
- returnValue.soundEffectModel =
31
- beat.soundEffectParams?.model ?? context.presentationStyle.soundEffectParams?.model ?? returnValue.soundEffectAgentInfo.defaultModel;
32
- returnValue.soundEffectFile = moviePaths.soundEffectFile;
33
- returnValue.soundEffectPrompt = beat.soundEffectPrompt;
28
+ const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
29
+ if (isMovie) {
30
+ if (beat.soundEffectPrompt) {
31
+ returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
32
+ returnValue.soundEffectModel =
33
+ beat.soundEffectParams?.model ?? context.presentationStyle.soundEffectParams?.model ?? returnValue.soundEffectAgentInfo.defaultModel;
34
+ returnValue.soundEffectFile = moviePaths.soundEffectFile;
35
+ returnValue.soundEffectPrompt = beat.soundEffectPrompt;
36
+ }
34
37
  }
35
38
  if (beat.enableLipSync) {
36
- returnValue.lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
37
- returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? returnValue.lipSyncAgentInfo.defaultModel;
39
+ const lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
40
+ returnValue.lipSyncAgentName = lipSyncAgentInfo.agentName;
41
+ returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? lipSyncAgentInfo.defaultModel;
38
42
  returnValue.lipSyncFile = moviePaths.lipSyncFile;
39
43
  // Audio file will be set from the beat's audio file when available
40
44
  returnValue.audioFile = studioBeat?.audioFile;
@@ -218,10 +218,11 @@ const beat_graph_data = {
218
218
  },
219
219
  lipSyncGenerator: {
220
220
  if: ":beat.enableLipSync",
221
- agent: ":preprocessor.lipSyncAgentInfo.agentName",
221
+ agent: ":preprocessor.lipSyncAgentName",
222
222
  inputs: {
223
223
  onComplete: [":soundEffectGenerator"], // to wait for soundEffectGenerator to finish
224
224
  movieFile: ":preprocessor.movieFile",
225
+ imageFile: ":preprocessor.referenceImageForMovie",
225
226
  audioFile: ":preprocessor.audioFile",
226
227
  lipSyncFile: ":preprocessor.lipSyncFile",
227
228
  params: {
@@ -1,9 +1,9 @@
1
1
  import "dotenv/config";
2
- import { GraphAI, assert } from "graphai";
2
+ import { GraphAI, assert, isNull } from "graphai";
3
3
  import * as agents from "@graphai/vanilla";
4
4
  import { openAIAgent } from "@graphai/openai_agent";
5
5
  import { fileWriteAgent } from "@graphai/vanilla_node_agents";
6
- import { recursiveSplitJa, replacementsJa, replacePairsJa } from "../utils/string.js";
6
+ import { recursiveSplitJa } from "../utils/string.js";
7
7
  import { settings2GraphAIConfig } from "../utils/utils.js";
8
8
  import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
9
9
  import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
@@ -13,17 +13,9 @@ const translateGraph = {
13
13
  version: 0.5,
14
14
  nodes: {
15
15
  context: {},
16
- defaultLang: {},
17
16
  outDirPath: {},
18
17
  outputMultilingualFilePath: {},
19
- lang: {
20
- agent: "stringUpdateTextAgent",
21
- inputs: {
22
- newText: ":context.studio.script.lang",
23
- oldText: ":defaultLang",
24
- },
25
- },
26
- targetLangs: {}, // TODO
18
+ targetLangs: {},
27
19
  mergeStudioResult: {
28
20
  isResult: true,
29
21
  agent: "mergeObjectAgent",
@@ -37,7 +29,6 @@ const translateGraph = {
37
29
  targetLangs: ":targetLangs",
38
30
  context: ":context",
39
31
  rows: ":context.studio.script.beats",
40
- lang: ":lang",
41
32
  },
42
33
  params: {
43
34
  rowKey: "beat",
@@ -62,7 +53,7 @@ const translateGraph = {
62
53
  beat: ":beat",
63
54
  multiLingual: ":multiLingual",
64
55
  rows: ":targetLangs",
65
- lang: ":lang.text",
56
+ lang: ":context.studio.script.lang",
66
57
  context: ":context",
67
58
  beatIndex: ":__mapIndex",
68
59
  },
@@ -120,17 +111,11 @@ const translateGraph = {
120
111
  },
121
112
  ttsTexts: {
122
113
  agent: (namedInputs) => {
123
- const { localizedText, targetLang } = namedInputs;
114
+ const { localizedText } = namedInputs;
124
115
  // cache
125
116
  if (localizedText.ttsTexts) {
126
117
  return localizedText;
127
118
  }
128
- if (targetLang === "ja") {
129
- return {
130
- ...localizedText,
131
- ttsTexts: localizedText?.texts?.map((text) => replacePairsJa(text, replacementsJa)),
132
- };
133
- }
134
119
  return {
135
120
  ...localizedText,
136
121
  ttsTexts: localizedText.texts,
@@ -180,18 +165,14 @@ const localizedTextCacheAgentFilter = async (context, next) => {
180
165
  if (!beat.text) {
181
166
  return { text: "" };
182
167
  }
183
- // The original text is unchanged and the target language text is present
184
- if (multiLingual.multiLingualTexts &&
185
- multiLingual.multiLingualTexts[lang] &&
186
- multiLingual.multiLingualTexts[lang].text === beat.text &&
187
- multiLingual.multiLingualTexts[targetLang] &&
188
- multiLingual.multiLingualTexts[targetLang].text) {
189
- return { text: multiLingual.multiLingualTexts[targetLang].text };
190
- }
191
168
  // same language
192
169
  if (targetLang === lang) {
193
170
  return { text: beat.text };
194
171
  }
172
+ // The original text is unchanged and the target language text is present
173
+ if (multiLingual.multiLingualTexts?.[lang]?.text === beat.text && multiLingual.multiLingualTexts[targetLang]?.text) {
174
+ return { text: multiLingual.multiLingualTexts[targetLang].text };
175
+ }
195
176
  try {
196
177
  MulmoStudioContextMethods.setBeatSessionState(mulmoContext, "multiLingual", beatIndex, true);
197
178
  return await next(context);
@@ -207,8 +188,6 @@ const agentFilters = [
207
188
  nodeIds: ["localizedTexts"],
208
189
  },
209
190
  ];
210
- const defaultLang = "en";
211
- const targetLangs = ["ja", "en"];
212
191
  export const translate = async (context, args) => {
213
192
  const { settings, callbacks } = args ?? {};
214
193
  try {
@@ -217,11 +196,14 @@ export const translate = async (context, args) => {
217
196
  const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
218
197
  const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
219
198
  mkdir(outDirPath);
199
+ const langs = (context.multiLingual ?? []).map((x) => Object.keys(x.multiLingualTexts)).flat(); // existing langs in multiLingual
200
+ const targetLangs = [
201
+ ...new Set([context.studio.script.lang, langs, context.lang, context.studio.script.captionParams?.lang].flat().filter((x) => !isNull(x))),
202
+ ];
220
203
  const config = settings2GraphAIConfig(settings, process.env);
221
204
  assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
222
205
  const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
223
206
  graph.injectValue("context", context);
224
- graph.injectValue("defaultLang", defaultLang);
225
207
  graph.injectValue("targetLangs", targetLangs);
226
208
  graph.injectValue("outDirPath", outDirPath);
227
209
  graph.injectValue("outputMultilingualFilePath", outputMultilingualFilePath);
@@ -3,7 +3,7 @@ import { GraphAILogger } from "graphai";
3
3
  import Replicate from "replicate";
4
4
  import { provider2LipSyncAgent } from "../utils/provider2agent.js";
5
5
  export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
6
- const { movieFile, audioFile } = namedInputs;
6
+ const { movieFile, audioFile, imageFile } = namedInputs;
7
7
  const apiKey = config?.apiKey;
8
8
  const model = params.model ?? provider2LipSyncAgent.replicate.defaultModel;
9
9
  if (!apiKey) {
@@ -12,10 +12,12 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
12
12
  const replicate = new Replicate({
13
13
  auth: apiKey,
14
14
  });
15
- const videoBuffer = readFileSync(movieFile);
15
+ const videoBuffer = movieFile ? readFileSync(movieFile) : undefined;
16
16
  const audioBuffer = readFileSync(audioFile);
17
- const videoUri = `data:video/quicktime;base64,${videoBuffer.toString("base64")}`;
17
+ const imageBuffer = imageFile ? readFileSync(imageFile) : undefined;
18
+ const videoUri = videoBuffer ? `data:video/quicktime;base64,${videoBuffer.toString("base64")}` : undefined;
18
19
  const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
20
+ const imageUri = imageBuffer ? `data:image/png;base64,${imageBuffer.toString("base64")}` : undefined;
19
21
  const input = {
20
22
  video: undefined,
21
23
  video_input: undefined,
@@ -23,6 +25,7 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
23
25
  audio: undefined,
24
26
  audio_input: undefined,
25
27
  audio_file: undefined,
28
+ image: undefined,
26
29
  };
27
30
  const modelParams = provider2LipSyncAgent.replicate.modelParams[model];
28
31
  if (!modelParams) {
@@ -30,12 +33,16 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
30
33
  }
31
34
  const videoParam = modelParams.video;
32
35
  const audioParam = modelParams.audio;
36
+ const imageParam = modelParams.image;
33
37
  if (videoParam === "video" || videoParam === "video_input" || videoParam === "video_url") {
34
38
  input[videoParam] = videoUri;
35
39
  }
36
40
  if (audioParam === "audio" || audioParam === "audio_input" || audioParam === "audio_file") {
37
41
  input[audioParam] = audioUri;
38
42
  }
43
+ if (imageParam === "image") {
44
+ input[imageParam] = imageUri;
45
+ }
39
46
  const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
40
47
  try {
41
48
  const output = await replicate.run(model_identifier, {
@@ -5,6 +5,6 @@ export const handler = async (argv) => {
5
5
  if (!context) {
6
6
  process.exit(1);
7
7
  }
8
- await runTranslateIfNeeded(context, argv);
8
+ await runTranslateIfNeeded(context);
9
9
  await audio(context);
10
10
  };
@@ -5,6 +5,6 @@ export const handler = async (argv) => {
5
5
  if (!context) {
6
6
  process.exit(1);
7
7
  }
8
- await runTranslateIfNeeded(context, argv);
8
+ await runTranslateIfNeeded(context);
9
9
  await images(context);
10
10
  };
@@ -5,6 +5,6 @@ export const handler = async (argv) => {
5
5
  if (!context) {
6
6
  process.exit(1);
7
7
  }
8
- await runTranslateIfNeeded(context, argv);
8
+ await runTranslateIfNeeded(context, true);
9
9
  await audio(context).then(images).then(captions).then(movie);
10
10
  };
@@ -5,7 +5,7 @@ export const handler = async (argv) => {
5
5
  if (!context) {
6
6
  process.exit(1);
7
7
  }
8
- await runTranslateIfNeeded(context, argv);
8
+ await runTranslateIfNeeded(context);
9
9
  await images(context);
10
10
  await pdf(context, argv.pdf_mode, argv.pdf_size);
11
11
  };
@@ -1,9 +1,6 @@
1
1
  import type { CliArgs } from "../types/cli_types.js";
2
2
  import { FileObject, InitOptions, MulmoStudioContext } from "../types/index.js";
3
- export declare const runTranslateIfNeeded: (context: MulmoStudioContext, argv: {
4
- l?: string;
5
- c?: string;
6
- }) => Promise<void>;
3
+ export declare const runTranslateIfNeeded: (context: MulmoStudioContext, includeCaption?: boolean) => Promise<void>;
7
4
  export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
8
5
  export declare const getFileObject: (args: {
9
6
  basedir?: string;
@@ -5,10 +5,11 @@ import clipboardy from "clipboardy";
5
5
  import { getBaseDirPath, getFullPath, getOutputStudioFilePath, resolveDirPath, mkdir, getOutputMultilingualFilePath, generateTimestampedFileName, } from "../utils/file.js";
6
6
  import { isHttp } from "../utils/utils.js";
7
7
  import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
8
+ import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
8
9
  import { translate } from "../actions/translate.js";
9
10
  import { initializeContextFromFiles } from "../utils/context.js";
10
- export const runTranslateIfNeeded = async (context, argv) => {
11
- if (argv.l || context.studio.script.captionParams?.lang) {
11
+ export const runTranslateIfNeeded = async (context, includeCaption = false) => {
12
+ if (MulmoStudioContextMethods.needTranslate(context, includeCaption)) {
12
13
  GraphAILogger.log("run translate");
13
14
  await translate(context);
14
15
  }
package/lib/mcp/server.js CHANGED
@@ -104,7 +104,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
104
104
  throw new Error("Failed to initialize context from MulmoScript");
105
105
  }
106
106
  // Run translation if needed
107
- await runTranslateIfNeeded(context, argv);
107
+ await runTranslateIfNeeded(context);
108
108
  // Execute the requested command
109
109
  switch (cmd) {
110
110
  case "movie":
@@ -36,9 +36,10 @@ export declare const MulmoPresentationStyleMethods: {
36
36
  defaultModel: import("../utils/provider2agent.js").ReplicateModel;
37
37
  models: import("../utils/provider2agent.js").ReplicateModel[];
38
38
  modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
39
- identifier?: `${string}/${string}:${string}`;
40
- video: string;
39
+ identifier?: `${string}/${string}:${string}` | `${string}/${string}`;
40
+ video?: string;
41
41
  audio: string;
42
+ image?: string;
42
43
  }>;
43
44
  };
44
45
  getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
@@ -18,6 +18,9 @@ const validators = [{ from: "1.0", to: "1.1", validator: validate_1_0 }];
18
18
  export const MulmoScriptMethods = {
19
19
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
20
20
  validate(script) {
21
+ const version = script.$mulmocast.version;
22
+ // lang was optional in 1.0 and 1.1
23
+ const defaultLang = version === "1.0" || version === "1.1" ? { lang: "en" } : {};
21
24
  const validatedScript = validators.reduce((acc, validator) => {
22
25
  if (acc.$mulmocast.version === validator.from) {
23
26
  const validated = validator.validator(acc);
@@ -25,7 +28,7 @@ export const MulmoScriptMethods = {
25
28
  return validated;
26
29
  }
27
30
  return acc;
28
- }, script);
31
+ }, { ...defaultLang, ...script });
29
32
  return mulmoScriptSchema.parse(validatedScript);
30
33
  },
31
34
  };
@@ -11,4 +11,5 @@ export declare const MulmoStudioContextMethods: {
11
11
  getCaption(context: MulmoStudioContext): string | undefined;
12
12
  setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
13
13
  setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
14
+ needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
14
15
  };
@@ -63,4 +63,12 @@ export const MulmoStudioContextMethods = {
63
63
  }
64
64
  notifyBeatStateChange(context, sessionType, index);
65
65
  },
66
+ needTranslate(context, includeCaption = false) {
67
+ // context.studio.script.lang = defaultLang, context.lang = targetLanguage.
68
+ if (includeCaption) {
69
+ return (context.studio.script.lang !== context.lang ||
70
+ (context.studio.script.captionParams?.lang && context.studio.script.lang !== context.studio.script.captionParams?.lang));
71
+ }
72
+ return context.studio.script.lang !== context.lang;
73
+ },
66
74
  };
@@ -77,6 +77,7 @@ export type LipSyncAgentInputs = {
77
77
  lipSyncFile: string;
78
78
  movieFile: string;
79
79
  audioFile: string;
80
+ imageFile: string;
80
81
  };
81
82
  export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
82
83
  export type ReplicateMovieAgentConfig = AgentConfig;
@@ -3861,7 +3861,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
3861
3861
  title?: string | undefined;
3862
3862
  description?: string | undefined;
3863
3863
  }>, "many">>;
3864
- lang: z.ZodOptional<z.ZodString>;
3864
+ lang: z.ZodString;
3865
3865
  beats: z.ZodArray<z.ZodObject<{
3866
3866
  speaker: z.ZodOptional<z.ZodString>;
3867
3867
  text: z.ZodDefault<z.ZodString>;
@@ -5036,6 +5036,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
5036
5036
  imagePath: z.ZodOptional<z.ZodString>;
5037
5037
  __test_invalid__: z.ZodOptional<z.ZodBoolean>;
5038
5038
  }, "strict", z.ZodTypeAny, {
5039
+ lang: string;
5039
5040
  imageParams: {
5040
5041
  provider: string;
5041
5042
  model?: string | undefined;
@@ -5332,7 +5333,6 @@ export declare const mulmoScriptSchema: z.ZodObject<{
5332
5333
  } | undefined;
5333
5334
  enableLipSync?: boolean | undefined;
5334
5335
  }[];
5335
- lang?: string | undefined;
5336
5336
  title?: string | undefined;
5337
5337
  description?: string | undefined;
5338
5338
  lipSyncParams?: {
@@ -5359,6 +5359,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
5359
5359
  imagePath?: string | undefined;
5360
5360
  __test_invalid__?: boolean | undefined;
5361
5361
  }, {
5362
+ lang: string;
5362
5363
  $mulmocast: {
5363
5364
  version: "1.1";
5364
5365
  credit?: "closing" | undefined;
@@ -5564,7 +5565,6 @@ export declare const mulmoScriptSchema: z.ZodObject<{
5564
5565
  } | undefined;
5565
5566
  enableLipSync?: boolean | undefined;
5566
5567
  }[];
5567
- lang?: string | undefined;
5568
5568
  title?: string | undefined;
5569
5569
  description?: string | undefined;
5570
5570
  imageParams?: {
@@ -6393,7 +6393,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
6393
6393
  title?: string | undefined;
6394
6394
  description?: string | undefined;
6395
6395
  }>, "many">>;
6396
- lang: z.ZodOptional<z.ZodString>;
6396
+ lang: z.ZodString;
6397
6397
  beats: z.ZodArray<z.ZodObject<{
6398
6398
  speaker: z.ZodOptional<z.ZodString>;
6399
6399
  text: z.ZodDefault<z.ZodString>;
@@ -7568,6 +7568,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
7568
7568
  imagePath: z.ZodOptional<z.ZodString>;
7569
7569
  __test_invalid__: z.ZodOptional<z.ZodBoolean>;
7570
7570
  }, "strict", z.ZodTypeAny, {
7571
+ lang: string;
7571
7572
  imageParams: {
7572
7573
  provider: string;
7573
7574
  model?: string | undefined;
@@ -7864,7 +7865,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
7864
7865
  } | undefined;
7865
7866
  enableLipSync?: boolean | undefined;
7866
7867
  }[];
7867
- lang?: string | undefined;
7868
7868
  title?: string | undefined;
7869
7869
  description?: string | undefined;
7870
7870
  lipSyncParams?: {
@@ -7891,6 +7891,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
7891
7891
  imagePath?: string | undefined;
7892
7892
  __test_invalid__?: boolean | undefined;
7893
7893
  }, {
7894
+ lang: string;
7894
7895
  $mulmocast: {
7895
7896
  version: "1.1";
7896
7897
  credit?: "closing" | undefined;
@@ -8096,7 +8097,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
8096
8097
  } | undefined;
8097
8098
  enableLipSync?: boolean | undefined;
8098
8099
  }[];
8099
- lang?: string | undefined;
8100
8100
  title?: string | undefined;
8101
8101
  description?: string | undefined;
8102
8102
  imageParams?: {
@@ -8275,6 +8275,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
8275
8275
  captionFile?: string | undefined;
8276
8276
  }[];
8277
8277
  script: {
8278
+ lang: string;
8278
8279
  imageParams: {
8279
8280
  provider: string;
8280
8281
  model?: string | undefined;
@@ -8571,7 +8572,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
8571
8572
  } | undefined;
8572
8573
  enableLipSync?: boolean | undefined;
8573
8574
  }[];
8574
- lang?: string | undefined;
8575
8575
  title?: string | undefined;
8576
8576
  description?: string | undefined;
8577
8577
  lipSyncParams?: {
@@ -8616,6 +8616,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
8616
8616
  captionFile?: string | undefined;
8617
8617
  }[];
8618
8618
  script: {
8619
+ lang: string;
8619
8620
  $mulmocast: {
8620
8621
  version: "1.1";
8621
8622
  credit?: "closing" | undefined;
@@ -8821,7 +8822,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
8821
8822
  } | undefined;
8822
8823
  enableLipSync?: boolean | undefined;
8823
8824
  }[];
8824
- lang?: string | undefined;
8825
8825
  title?: string | undefined;
8826
8826
  description?: string | undefined;
8827
8827
  imageParams?: {
@@ -371,7 +371,7 @@ export const mulmoScriptSchema = mulmoPresentationStyleSchema
371
371
  title: z.string().optional(),
372
372
  description: z.string().optional(),
373
373
  references: z.array(mulmoReferenceSchema).optional(),
374
- lang: langSchema.optional(), // default "en"
374
+ lang: langSchema, // required (default WAS "en")
375
375
  beats: z.array(mulmoBeatSchema).min(1),
376
376
  // TODO: Delete it later
377
377
  imagePath: z.string().optional(), // for keynote images movie ??
@@ -53,7 +53,7 @@ export type FileDirs = {
53
53
  export type MulmoStudioContext = {
54
54
  fileDirs: FileDirs;
55
55
  studio: MulmoStudio;
56
- lang?: string;
56
+ lang: string;
57
57
  force: boolean;
58
58
  sessionState: MulmoSessionState;
59
59
  presentationStyle: MulmoPresentationStyle;
@@ -4,7 +4,7 @@ export const imageDirName = "images";
4
4
  export const cacheDirName = "cache";
5
5
  export const pdf_modes = ["slide", "talk", "handout"];
6
6
  export const pdf_sizes = ["letter", "a4"];
7
- export const languages = ["en", "ja"];
7
+ export const languages = ["en", "ja", "fr", "es"];
8
8
  export const storyToScriptGenerateMode = {
9
9
  stepWise: "step_wise",
10
10
  oneStep: "one_step",