mulmocast 1.2.10 → 1.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/actions/audio.d.ts +3 -1
- package/lib/actions/audio.js +45 -8
- package/lib/actions/image_references.js +3 -3
- package/lib/actions/translate.js +13 -5
- package/lib/agents/image_genai_agent.js +47 -17
- package/lib/index.common.d.ts +5 -1
- package/lib/index.common.js +5 -1
- package/lib/methods/mulmo_media_source.js +2 -3
- package/lib/methods/mulmo_presentation_style.d.ts +5 -1
- package/lib/methods/mulmo_presentation_style.js +5 -1
- package/lib/methods/mulmo_script.d.ts +5 -0
- package/lib/methods/mulmo_script.js +6 -1
- package/lib/methods/mulmo_studio_context.d.ts +5 -1
- package/lib/methods/mulmo_studio_context.js +6 -5
- package/lib/types/agent.d.ts +3 -1
- package/lib/utils/context.js +1 -2
- package/lib/utils/file.d.ts +1 -0
- package/lib/utils/file.js +4 -0
- package/lib/utils/filters.js +1 -1
- package/lib/utils/provider2agent.js +1 -1
- package/lib/utils/utils.d.ts +7 -2
- package/lib/utils/utils.js +9 -5
- package/lib/utils/utils_node.d.ts +1 -0
- package/lib/utils/utils_node.js +4 -0
- package/package.json +1 -6
- package/scripts/test/test_genai.json +8 -0
- package/scripts/test/test_image_refs.json +10 -0
package/lib/actions/audio.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
2
|
import { MulmoStudioContext, MulmoBeat, PublicAPIArgs } from "../types/index.js";
|
|
3
3
|
export declare const getBeatAudioPath: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
|
|
4
|
-
export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, args?: PublicAPIArgs
|
|
4
|
+
export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, args?: PublicAPIArgs & {
|
|
5
|
+
langs: string[];
|
|
6
|
+
}) => Promise<void>;
|
|
5
7
|
export declare const audio: (context: MulmoStudioContext, args?: PublicAPIArgs) => Promise<MulmoStudioContext>;
|
package/lib/actions/audio.js
CHANGED
|
@@ -6,7 +6,8 @@ import { ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsElevenlabsAgent,
|
|
|
6
6
|
import { text2SpeechProviderSchema } from "../types/index.js";
|
|
7
7
|
import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
|
|
8
8
|
import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
|
|
9
|
-
import {
|
|
9
|
+
import { localizedText, settings2GraphAIConfig } from "../utils/utils.js";
|
|
10
|
+
import { text2hash } from "../utils/utils_node.js";
|
|
10
11
|
import { provider2TTSAgent } from "../utils/provider2agent.js";
|
|
11
12
|
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
12
13
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
@@ -39,9 +40,9 @@ export const getBeatAudioPath = (text, context, beat, lang) => {
|
|
|
39
40
|
const audioFile = getAudioFilePath(audioDirPath, context.studio.filename, audioFileName, lang);
|
|
40
41
|
return getAudioPath(context, beat, audioFile);
|
|
41
42
|
};
|
|
42
|
-
const
|
|
43
|
-
const { beat, studioBeat, multiLingual, context } = namedInputs;
|
|
44
|
-
const { lang } = context;
|
|
43
|
+
const preprocessorAgent = (namedInputs) => {
|
|
44
|
+
const { beat, studioBeat, multiLingual, context, lang } = namedInputs;
|
|
45
|
+
// const { lang } = context;
|
|
45
46
|
const text = localizedText(beat, multiLingual, lang);
|
|
46
47
|
const { voiceId, provider, speechOptions, model } = getAudioParam(context, beat);
|
|
47
48
|
const audioPath = getBeatAudioPath(text, context, beat, lang);
|
|
@@ -67,13 +68,15 @@ const graph_tts = {
|
|
|
67
68
|
multiLingual: {},
|
|
68
69
|
context: {},
|
|
69
70
|
__mapIndex: {},
|
|
71
|
+
lang: {},
|
|
70
72
|
preprocessor: {
|
|
71
|
-
agent:
|
|
73
|
+
agent: preprocessorAgent,
|
|
72
74
|
inputs: {
|
|
73
75
|
beat: ":beat",
|
|
74
76
|
studioBeat: ":studioBeat",
|
|
75
77
|
multiLingual: ":multiLingual",
|
|
76
78
|
context: ":context",
|
|
79
|
+
lang: ":lang",
|
|
77
80
|
},
|
|
78
81
|
},
|
|
79
82
|
tts: {
|
|
@@ -102,6 +105,33 @@ const graph_tts = {
|
|
|
102
105
|
},
|
|
103
106
|
},
|
|
104
107
|
};
|
|
108
|
+
const graph_tts_map = {
|
|
109
|
+
version: 0.5,
|
|
110
|
+
concurrency: 8,
|
|
111
|
+
nodes: {
|
|
112
|
+
beat: {},
|
|
113
|
+
studioBeat: {},
|
|
114
|
+
multiLingual: {},
|
|
115
|
+
context: {},
|
|
116
|
+
__mapIndex: {},
|
|
117
|
+
langs: {},
|
|
118
|
+
map: {
|
|
119
|
+
agent: "mapAgent",
|
|
120
|
+
inputs: {
|
|
121
|
+
rows: ":langs",
|
|
122
|
+
beat: ":beat",
|
|
123
|
+
studioBeat: ":studioBeat",
|
|
124
|
+
multiLingual: ":multiLingual",
|
|
125
|
+
context: ":context",
|
|
126
|
+
__mapIndex: ":__mapIndex",
|
|
127
|
+
},
|
|
128
|
+
params: {
|
|
129
|
+
rowKey: "lang",
|
|
130
|
+
},
|
|
131
|
+
graph: graph_tts,
|
|
132
|
+
},
|
|
133
|
+
},
|
|
134
|
+
};
|
|
105
135
|
const graph_data = {
|
|
106
136
|
version: 0.5,
|
|
107
137
|
concurrency: 8,
|
|
@@ -118,6 +148,7 @@ const graph_data = {
|
|
|
118
148
|
studioBeat: ":context.studio.beats",
|
|
119
149
|
multiLingual: ":context.multiLingual",
|
|
120
150
|
context: ":context",
|
|
151
|
+
lang: ":context.lang",
|
|
121
152
|
},
|
|
122
153
|
params: {
|
|
123
154
|
rowKey: "beat",
|
|
@@ -187,7 +218,7 @@ const audioAgents = {
|
|
|
187
218
|
combineAudioFilesAgent,
|
|
188
219
|
};
|
|
189
220
|
export const generateBeatAudio = async (index, context, args) => {
|
|
190
|
-
const { settings, callbacks } = args ?? {};
|
|
221
|
+
const { settings, callbacks, langs } = args ?? {};
|
|
191
222
|
try {
|
|
192
223
|
MulmoStudioContextMethods.setSessionState(context, "audio", true);
|
|
193
224
|
const fileName = MulmoStudioContextMethods.getFileName(context);
|
|
@@ -198,12 +229,18 @@ export const generateBeatAudio = async (index, context, args) => {
|
|
|
198
229
|
mkdir(audioSegmentDirPath);
|
|
199
230
|
const config = settings2GraphAIConfig(settings);
|
|
200
231
|
const taskManager = new TaskManager(getConcurrency(context));
|
|
201
|
-
const graph = new GraphAI(graph_tts, audioAgents, { agentFilters, taskManager, config });
|
|
232
|
+
const graph = new GraphAI(langs ? graph_tts_map : graph_tts, audioAgents, { agentFilters, taskManager, config });
|
|
202
233
|
graph.injectValue("__mapIndex", index);
|
|
203
234
|
graph.injectValue("beat", context.studio.script.beats[index]);
|
|
204
235
|
graph.injectValue("studioBeat", context.studio.beats[index]);
|
|
205
|
-
graph.injectValue("multiLingual", context.multiLingual);
|
|
236
|
+
graph.injectValue("multiLingual", context.multiLingual[index]);
|
|
206
237
|
graph.injectValue("context", context);
|
|
238
|
+
if (langs) {
|
|
239
|
+
graph.injectValue("langs", langs);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
graph.injectValue("lang", context.lang);
|
|
243
|
+
}
|
|
207
244
|
if (callbacks) {
|
|
208
245
|
callbacks.forEach((callback) => {
|
|
209
246
|
graph.registerCallback(callback);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import { GraphAI, GraphAILogger } from "graphai";
|
|
3
|
-
import { getReferenceImagePath } from "../utils/file.js";
|
|
3
|
+
import { getReferenceImagePath, resolveAssetPath } from "../utils/file.js";
|
|
4
4
|
import { getExtention } from "../utils/utils.js";
|
|
5
5
|
import { graphOption } from "./images.js";
|
|
6
|
-
import { MulmoPresentationStyleMethods
|
|
6
|
+
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
7
7
|
import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent } from "../agents/index.js";
|
|
8
8
|
// public api
|
|
9
9
|
// Application may call this function directly to generate reference image.
|
|
@@ -70,7 +70,7 @@ export const getImageRefs = async (context) => {
|
|
|
70
70
|
}
|
|
71
71
|
else if (image.type === "image") {
|
|
72
72
|
if (image.source.kind === "path") {
|
|
73
|
-
imageRefs[key] =
|
|
73
|
+
imageRefs[key] = resolveAssetPath(context, image.source.path);
|
|
74
74
|
}
|
|
75
75
|
else if (image.source.kind === "url") {
|
|
76
76
|
imageRefs[key] = await downLoadImage(context, key, image.source.url);
|
package/lib/actions/translate.js
CHANGED
|
@@ -5,8 +5,7 @@ import * as agents from "@graphai/vanilla";
|
|
|
5
5
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
6
6
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
7
7
|
import { splitText } from "../utils/string.js";
|
|
8
|
-
import { settings2GraphAIConfig, multiLingualObjectToArray } from "../utils/utils.js";
|
|
9
|
-
import { beatId } from "../utils/common.js";
|
|
8
|
+
import { settings2GraphAIConfig, beatId, multiLingualObjectToArray } from "../utils/utils.js";
|
|
10
9
|
import { getMultiLingual } from "../utils/context.js";
|
|
11
10
|
import { currentMulmoScriptVersion } from "../utils/const.js";
|
|
12
11
|
import { getOutputMultilingualFilePath, mkdir, writingMessage, hashSHA256 } from "../utils/file.js";
|
|
@@ -145,10 +144,18 @@ const translateGraph = {
|
|
|
145
144
|
mergeStudioResult: {
|
|
146
145
|
isResult: true,
|
|
147
146
|
agent: (namedInputs) => {
|
|
148
|
-
const { multiLingual, beats } = namedInputs;
|
|
147
|
+
const { multiLingual, beats, originalMultiLingual } = namedInputs;
|
|
149
148
|
const multiLingualObject = beats.reduce((tmp, beat, beatIndex) => {
|
|
150
149
|
const key = beatId(beat?.id, beatIndex);
|
|
151
|
-
|
|
150
|
+
const originalData = originalMultiLingual[beatIndex]?.multiLingualTexts ?? {};
|
|
151
|
+
const { multiLingualTexts, cacheKey } = multiLingual[beatIndex];
|
|
152
|
+
tmp[key] = {
|
|
153
|
+
cacheKey,
|
|
154
|
+
multiLingualTexts: {
|
|
155
|
+
...originalData,
|
|
156
|
+
...multiLingualTexts,
|
|
157
|
+
},
|
|
158
|
+
};
|
|
152
159
|
return tmp;
|
|
153
160
|
}, {});
|
|
154
161
|
return {
|
|
@@ -157,7 +164,8 @@ const translateGraph = {
|
|
|
157
164
|
};
|
|
158
165
|
},
|
|
159
166
|
inputs: {
|
|
160
|
-
|
|
167
|
+
originalMultiLingual: ":context.multiLingual", // original
|
|
168
|
+
multiLingual: ":beatsMap.mergeMultiLingualData", // update
|
|
161
169
|
beats: ":context.studio.script.beats",
|
|
162
170
|
},
|
|
163
171
|
},
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import fs from "fs";
|
|
1
2
|
import { GraphAILogger } from "graphai";
|
|
2
3
|
import { getAspectRatio } from "./movie_google_agent.js";
|
|
3
4
|
import { provider2ImageAgent } from "../utils/provider2agent.js";
|
|
4
5
|
import { GoogleGenAI, PersonGeneration } from "@google/genai";
|
|
5
6
|
export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
6
|
-
const { prompt } = namedInputs;
|
|
7
|
+
const { prompt, referenceImages } = namedInputs;
|
|
7
8
|
const aspectRatio = getAspectRatio(params.canvasSize);
|
|
8
9
|
const model = params.model ?? provider2ImageAgent["google"].defaultModel;
|
|
9
10
|
const apiKey = config?.apiKey;
|
|
@@ -12,24 +13,53 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
12
13
|
}
|
|
13
14
|
try {
|
|
14
15
|
const ai = new GoogleGenAI({ apiKey });
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
16
|
+
if (model === "gemini-2.5-flash-image-preview") {
|
|
17
|
+
const contents = [{ text: prompt }];
|
|
18
|
+
referenceImages?.forEach((imagePath) => {
|
|
19
|
+
const imageData = fs.readFileSync(imagePath);
|
|
20
|
+
const base64Image = imageData.toString("base64");
|
|
21
|
+
contents.push({ inlineData: { mimeType: "image/png", data: base64Image } });
|
|
22
|
+
});
|
|
23
|
+
// NOTE: There is no way to specify the aspect ratio for Gemini.
|
|
24
|
+
const response = await ai.models.generateContent({ model, contents });
|
|
25
|
+
if (!response.candidates?.[0]?.content?.parts) {
|
|
26
|
+
throw new Error("ERROR: generateContent returned no candidates");
|
|
27
|
+
}
|
|
28
|
+
for (const part of response.candidates[0].content.parts) {
|
|
29
|
+
if (part.text) {
|
|
30
|
+
GraphAILogger.info("Gemini image generation response:", part.text);
|
|
31
|
+
}
|
|
32
|
+
else if (part.inlineData) {
|
|
33
|
+
const imageData = part.inlineData.data;
|
|
34
|
+
if (!imageData) {
|
|
35
|
+
throw new Error("ERROR: generateContent returned no image data");
|
|
36
|
+
}
|
|
37
|
+
const buffer = Buffer.from(imageData, "base64");
|
|
38
|
+
return { buffer };
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
throw new Error("ERROR: generateContent returned no image data");
|
|
27
42
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
43
|
+
else {
|
|
44
|
+
const response = await ai.models.generateImages({
|
|
45
|
+
model,
|
|
46
|
+
prompt,
|
|
47
|
+
config: {
|
|
48
|
+
numberOfImages: 1, // default is 4!
|
|
49
|
+
aspectRatio,
|
|
50
|
+
personGeneration: PersonGeneration.ALLOW_ALL,
|
|
51
|
+
// safetyFilterLevel: SafetyFilterLevel.BLOCK_ONLY_HIGH,
|
|
52
|
+
},
|
|
53
|
+
});
|
|
54
|
+
if (!response.generatedImages || response.generatedImages.length === 0) {
|
|
55
|
+
throw new Error("ERROR: generateImage returned no generated images");
|
|
56
|
+
}
|
|
57
|
+
const image = response.generatedImages[0].image;
|
|
58
|
+
if (image && image.imageBytes) {
|
|
59
|
+
return { buffer: Buffer.from(image.imageBytes, "base64") };
|
|
60
|
+
}
|
|
61
|
+
throw new Error("ERROR: generateImage returned no image bytes");
|
|
31
62
|
}
|
|
32
|
-
throw new Error("ERROR: generateImage returned no image bytes");
|
|
33
63
|
}
|
|
34
64
|
catch (error) {
|
|
35
65
|
GraphAILogger.info("Failed to generate image:", error);
|
package/lib/index.common.d.ts
CHANGED
|
@@ -2,4 +2,8 @@ export * from "./types/index.js";
|
|
|
2
2
|
export * from "./utils/provider2agent.js";
|
|
3
3
|
export * from "./utils/const.js";
|
|
4
4
|
export * from "./utils/string.js";
|
|
5
|
-
export * from "./utils/
|
|
5
|
+
export * from "./utils/utils.js";
|
|
6
|
+
export * from "./utils/prompt.js";
|
|
7
|
+
export * from "./methods/mulmo_presentation_style.js";
|
|
8
|
+
export * from "./methods/mulmo_script.js";
|
|
9
|
+
export * from "./methods/mulmo_studio_context.js";
|
package/lib/index.common.js
CHANGED
|
@@ -3,4 +3,8 @@ export * from "./types/index.js";
|
|
|
3
3
|
export * from "./utils/provider2agent.js";
|
|
4
4
|
export * from "./utils/const.js";
|
|
5
5
|
export * from "./utils/string.js";
|
|
6
|
-
export * from "./utils/
|
|
6
|
+
export * from "./utils/utils.js";
|
|
7
|
+
export * from "./utils/prompt.js";
|
|
8
|
+
export * from "./methods/mulmo_presentation_style.js";
|
|
9
|
+
export * from "./methods/mulmo_script.js";
|
|
10
|
+
export * from "./methods/mulmo_studio_context.js";
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
|
-
import { getFullPath } from "../utils/file.js";
|
|
3
|
-
import { MulmoStudioContextMethods } from "../methods/index.js";
|
|
2
|
+
import { getFullPath, resolveAssetPath } from "../utils/file.js";
|
|
4
3
|
export const MulmoMediaSourceMethods = {
|
|
5
4
|
async getText(mediaSource, context) {
|
|
6
5
|
if (mediaSource.kind === "text") {
|
|
@@ -23,7 +22,7 @@ export const MulmoMediaSourceMethods = {
|
|
|
23
22
|
if (!mediaSource)
|
|
24
23
|
return null;
|
|
25
24
|
if (mediaSource.kind === "path") {
|
|
26
|
-
return
|
|
25
|
+
return resolveAssetPath(context, mediaSource.path);
|
|
27
26
|
}
|
|
28
27
|
if (mediaSource.kind === "url") {
|
|
29
28
|
return mediaSource.url;
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
2
6
|
import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext } from "../types/index.js";
|
|
3
7
|
export declare const MulmoPresentationStyleMethods: {
|
|
4
8
|
getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
2
6
|
import { isNull } from "graphai";
|
|
3
7
|
import { userAssert } from "../utils/utils.js";
|
|
4
8
|
import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
1
6
|
import { type MulmoStudioBeat, type MulmoScript, type MulmoStudioMultiLingual } from "../types/index.js";
|
|
2
7
|
export declare const MulmoScriptMethods: {
|
|
3
8
|
validate(script: any): MulmoScript;
|
|
@@ -1,6 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
1
6
|
import { GraphAILogger } from "graphai";
|
|
2
7
|
import { mulmoScriptSchema, mulmoStudioMultiLingualFileSchema } from "../types/index.js";
|
|
3
|
-
import { beatId } from "../utils/
|
|
8
|
+
import { beatId } from "../utils/utils.js";
|
|
4
9
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
5
10
|
const validate_1_0 = (script) => {
|
|
6
11
|
if (script.speechParams?.provider) {
|
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
1
6
|
import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType } from "../types/index.js";
|
|
2
7
|
export declare const addSessionProgressCallback: (cb: SessionProgressCallback) => void;
|
|
3
8
|
export declare const removeSessionProgressCallback: (cb: SessionProgressCallback) => void;
|
|
4
9
|
export declare const MulmoStudioContextMethods: {
|
|
5
|
-
resolveAssetPath(context: MulmoStudioContext, relativePath: string): string;
|
|
6
10
|
getAudioDirPath(context: MulmoStudioContext): string;
|
|
7
11
|
getImageDirPath(context: MulmoStudioContext): string;
|
|
8
12
|
getImageProjectDirPath(context: MulmoStudioContext): string;
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
6
|
+
import { beatId } from "../utils/utils.js";
|
|
3
7
|
import { GraphAILogger } from "graphai";
|
|
4
8
|
const sessionProgressCallbacks = new Set();
|
|
5
9
|
export const addSessionProgressCallback = (cb) => {
|
|
@@ -25,9 +29,6 @@ const notifyBeatStateChange = (context, sessionType, id) => {
|
|
|
25
29
|
}
|
|
26
30
|
};
|
|
27
31
|
export const MulmoStudioContextMethods = {
|
|
28
|
-
resolveAssetPath(context, relativePath) {
|
|
29
|
-
return path.resolve(context.fileDirs.mulmoFileDirPath, relativePath);
|
|
30
|
-
},
|
|
31
32
|
getAudioDirPath(context) {
|
|
32
33
|
return context.fileDirs.audioDirPath;
|
|
33
34
|
},
|
package/lib/types/agent.d.ts
CHANGED
|
@@ -27,7 +27,9 @@ export type AgentErrorResult = {
|
|
|
27
27
|
export type AgentConfig = {
|
|
28
28
|
apiKey?: string;
|
|
29
29
|
};
|
|
30
|
-
export type ImageAgentInputs = AgentPromptInputs
|
|
30
|
+
export type ImageAgentInputs = AgentPromptInputs & {
|
|
31
|
+
referenceImages: string[] | null | undefined;
|
|
32
|
+
};
|
|
31
33
|
export type OpenAIImageAgentInputs = AgentPromptInputs & {
|
|
32
34
|
referenceImages: string[] | null | undefined;
|
|
33
35
|
};
|
package/lib/utils/context.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import fs from "fs";
|
|
3
3
|
import { readMulmoScriptFile, fetchMulmoScriptFile } from "./file.js";
|
|
4
|
-
import { multiLingualObjectToArray } from "./utils.js";
|
|
5
|
-
import { beatId } from "./common.js";
|
|
4
|
+
import { beatId, multiLingualObjectToArray } from "./utils.js";
|
|
6
5
|
import { mulmoStudioSchema, mulmoCaptionParamsSchema, mulmoPresentationStyleSchema } from "../types/schema.js";
|
|
7
6
|
import { MulmoPresentationStyleMethods, MulmoScriptMethods, MulmoStudioMultiLingualMethod } from "../methods/index.js";
|
|
8
7
|
const mulmoCredit = (speaker) => {
|
package/lib/utils/file.d.ts
CHANGED
|
@@ -38,6 +38,7 @@ export declare const getCaptionImagePath: (context: MulmoStudioContext, index: n
|
|
|
38
38
|
export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
|
|
39
39
|
export declare const getPromptTemplateFilePath: (promptTemplateName: string) => string;
|
|
40
40
|
export declare const mkdir: (dirPath: string) => void;
|
|
41
|
+
export declare const resolveAssetPath: (context: MulmoStudioContext, relativePath: string) => string;
|
|
41
42
|
export declare const silent60secPath: () => string;
|
|
42
43
|
export declare const defaultBGMPath: () => string;
|
|
43
44
|
export declare const mulmoCreditPath: () => string;
|
package/lib/utils/file.js
CHANGED
|
@@ -128,6 +128,10 @@ export const mkdir = (dirPath) => {
|
|
|
128
128
|
fs.mkdirSync(dirPath, { recursive: true });
|
|
129
129
|
}
|
|
130
130
|
};
|
|
131
|
+
// asset path
|
|
132
|
+
export const resolveAssetPath = (context, relativePath) => {
|
|
133
|
+
return path.resolve(context.fileDirs.mulmoFileDirPath, relativePath);
|
|
134
|
+
};
|
|
131
135
|
// export const silentPath = path.resolve(npmRoot, "./assets/audio/silent300.mp3");
|
|
132
136
|
// export const silentLastPath = path.resolve(npmRoot, "./assets/audio/silent800.mp3");
|
|
133
137
|
export const silent60secPath = () => path.resolve(npmRoot, "./assets/audio/silent60sec.mp3");
|
package/lib/utils/filters.js
CHANGED
|
@@ -4,7 +4,7 @@ import path from "path";
|
|
|
4
4
|
import fsPromise from "fs/promises";
|
|
5
5
|
import { GraphAILogger } from "graphai";
|
|
6
6
|
import { writingMessage } from "./file.js";
|
|
7
|
-
import { text2hash } from "./
|
|
7
|
+
import { text2hash } from "./utils_node.js";
|
|
8
8
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
9
9
|
import { replacementsJa, replacePairsJa } from "../utils/string.js";
|
|
10
10
|
export const nijovoiceTextAgentFilter = async (context, next) => {
|
|
@@ -38,7 +38,7 @@ export const provider2ImageAgent = {
|
|
|
38
38
|
google: {
|
|
39
39
|
agentName: "imageGenAIAgent",
|
|
40
40
|
defaultModel: "imagen-4.0-generate-preview-06-06",
|
|
41
|
-
models: ["imagen-3.0-generate-002", "imagen-4.0-generate-preview-06-06", "imagen-4.0-ultra-generate-preview-06-06"],
|
|
41
|
+
models: ["imagen-3.0-generate-002", "imagen-4.0-generate-preview-06-06", "imagen-4.0-ultra-generate-preview-06-06", "gemini-2.5-flash-image-preview"],
|
|
42
42
|
},
|
|
43
43
|
mock: {
|
|
44
44
|
agentName: "mediaMockAgent",
|
package/lib/utils/utils.d.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
1
6
|
import type { ConfigDataDictionary, DefaultConfigData } from "graphai";
|
|
2
7
|
import { MulmoBeat, MulmoStudioBeat, MulmoStudioMultiLingual, MulmoStudioMultiLingualData } from "../types/index.js";
|
|
3
|
-
import type
|
|
8
|
+
import { type LLM } from "./provider2agent.js";
|
|
4
9
|
export declare const llmPair: (_llm?: LLM, _model?: string) => {
|
|
5
10
|
agent: "mediaMockAgent" | "openAIAgent" | "anthropicAgent" | "geminiAgent" | "groqAgent";
|
|
6
11
|
model: string;
|
|
@@ -8,7 +13,6 @@ export declare const llmPair: (_llm?: LLM, _model?: string) => {
|
|
|
8
13
|
};
|
|
9
14
|
export declare const chunkArray: <T>(array: T[], size?: number) => T[][];
|
|
10
15
|
export declare const isHttp: (fileOrUrl: string) => boolean;
|
|
11
|
-
export declare const text2hash: (input: string) => string;
|
|
12
16
|
export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoStudioMultiLingualData, targetLang?: string, defaultLang?: string) => string;
|
|
13
17
|
export declare function userAssert(condition: boolean, message: string): asserts condition;
|
|
14
18
|
export declare const settings2GraphAIConfig: (settings?: Record<string, string>, env?: Record<string, string | undefined>) => ConfigDataDictionary<DefaultConfigData>;
|
|
@@ -19,6 +23,7 @@ type CleanableObject = {
|
|
|
19
23
|
[key: string]: CleanableValue;
|
|
20
24
|
};
|
|
21
25
|
export declare const deepClean: <T extends CleanableValue>(input: T) => T | undefined;
|
|
26
|
+
export declare const beatId: (id: string | undefined, index: number) => string;
|
|
22
27
|
export declare const multiLingualObjectToArray: (multiLingual: MulmoStudioMultiLingual | undefined, beats: MulmoStudioBeat[]) => {
|
|
23
28
|
multiLingualTexts: Record<string, {
|
|
24
29
|
text: string;
|
package/lib/utils/utils.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Browser-friendly packages only.
|
|
3
|
+
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
|
+
* Works in both Node.js and modern browsers.
|
|
5
|
+
*/
|
|
2
6
|
import { provider2LLMAgent } from "./provider2agent.js";
|
|
3
|
-
import { beatId } from "./common.js";
|
|
4
7
|
export const llmPair = (_llm, _model) => {
|
|
5
8
|
const llmKey = _llm ?? "openai";
|
|
6
9
|
const agent = provider2LLMAgent[llmKey]?.agentName ?? provider2LLMAgent.openai.agentName;
|
|
@@ -18,9 +21,6 @@ export const chunkArray = (array, size = 3) => {
|
|
|
18
21
|
export const isHttp = (fileOrUrl) => {
|
|
19
22
|
return /^https?:\/\//.test(fileOrUrl);
|
|
20
23
|
};
|
|
21
|
-
export const text2hash = (input) => {
|
|
22
|
-
return crypto.createHash("sha256").update(input).digest("hex");
|
|
23
|
-
};
|
|
24
24
|
export const localizedText = (beat, multiLingualData, targetLang, defaultLang) => {
|
|
25
25
|
if (targetLang === defaultLang) {
|
|
26
26
|
return beat.text;
|
|
@@ -118,6 +118,10 @@ export const deepClean = (input) => {
|
|
|
118
118
|
}
|
|
119
119
|
return input;
|
|
120
120
|
};
|
|
121
|
+
export const beatId = (id, index) => {
|
|
122
|
+
const key = id ?? `__index__${index}`;
|
|
123
|
+
return key;
|
|
124
|
+
};
|
|
121
125
|
export const multiLingualObjectToArray = (multiLingual, beats) => {
|
|
122
126
|
return beats.map((beat, index) => {
|
|
123
127
|
const key = beatId(beat?.id, index);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const text2hash: (input: string) => string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.12",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.node.js",
|
|
@@ -55,7 +55,6 @@
|
|
|
55
55
|
"format": "prettier --write '{src,scripts,assets/templates,assets/styles,draft,ideason,scripts_mag2,proto,test,batch,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'",
|
|
56
56
|
"deep_research": "npx tsx ./src/tools/deep_research.ts",
|
|
57
57
|
"template": "npx tsx batch/template2tsobject.ts && yarn run format",
|
|
58
|
-
"fake_data": "npx tsx test/fake/sample.ts",
|
|
59
58
|
"mcp_server": "npx tsx ./src/mcp/server.ts"
|
|
60
59
|
},
|
|
61
60
|
"repository": "git+ssh://git@github.com/receptron/mulmocast-cli.git",
|
|
@@ -81,7 +80,6 @@
|
|
|
81
80
|
"@inquirer/select": "^4.3.2",
|
|
82
81
|
"@modelcontextprotocol/sdk": "^1.17.4",
|
|
83
82
|
"@tavily/core": "^0.5.11",
|
|
84
|
-
"canvas": "^3.2.0",
|
|
85
83
|
"clipboardy": "^4.0.0",
|
|
86
84
|
"dotenv": "^17.2.1",
|
|
87
85
|
"fluent-ffmpeg": "^2.1.3",
|
|
@@ -96,8 +94,6 @@
|
|
|
96
94
|
"zod-to-json-schema": "^3.24.6"
|
|
97
95
|
},
|
|
98
96
|
"devDependencies": {
|
|
99
|
-
"@anatine/zod-mock": "^3.14.0",
|
|
100
|
-
"@faker-js/faker": "^9.9.0",
|
|
101
97
|
"@receptron/test_utils": "^2.0.3",
|
|
102
98
|
"@types/fluent-ffmpeg": "^2.1.26",
|
|
103
99
|
"@types/yargs": "^17.0.33",
|
|
@@ -106,7 +102,6 @@
|
|
|
106
102
|
"eslint-plugin-prettier": "^5.5.4",
|
|
107
103
|
"eslint-plugin-sonarjs": "^3.0.5",
|
|
108
104
|
"prettier": "^3.6.2",
|
|
109
|
-
"ts-node": "^10.9.2",
|
|
110
105
|
"tsx": "^4.20.5",
|
|
111
106
|
"typescript": "^5.9.2",
|
|
112
107
|
"typescript-eslint": "^8.41.0"
|
|
@@ -9,6 +9,14 @@
|
|
|
9
9
|
},
|
|
10
10
|
"lang": "en",
|
|
11
11
|
"beats": [
|
|
12
|
+
{
|
|
13
|
+
"id": "gemini_2_5_flash_image_preview",
|
|
14
|
+
"text": "image generated by gemini-2.5-flash-image-preview",
|
|
15
|
+
"imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
16
|
+
"imageParams": {
|
|
17
|
+
"model": "gemini-2.5-flash-image-preview"
|
|
18
|
+
}
|
|
19
|
+
},
|
|
12
20
|
{
|
|
13
21
|
"id": "imagen_3",
|
|
14
22
|
"text": "image generated by imagen-3",
|
|
@@ -45,6 +45,16 @@
|
|
|
45
45
|
"text": "Hello World with no reference image",
|
|
46
46
|
"imagePrompt": "Saying hello to the world",
|
|
47
47
|
"imageNames": []
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"id": "gemini_2_5_flash_image_preview",
|
|
51
|
+
"text": "Hello World with a witch and a broom with Gemini",
|
|
52
|
+
"imagePrompt": "Saying hello to the world",
|
|
53
|
+
"imageNames": ["witch", "broom"],
|
|
54
|
+
"imageParams": {
|
|
55
|
+
"provider": "google",
|
|
56
|
+
"model": "gemini-2.5-flash-image-preview"
|
|
57
|
+
}
|
|
48
58
|
}
|
|
49
59
|
]
|
|
50
60
|
}
|