mulmocast 1.1.9 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/images/mulmocast_credit.png +0 -0
- package/lib/actions/audio.js +4 -20
- package/lib/actions/image_references.js +3 -2
- package/lib/actions/images.js +4 -0
- package/lib/actions/translate.d.ts +4 -0
- package/lib/actions/translate.js +10 -13
- package/lib/agents/index.d.ts +2 -1
- package/lib/agents/index.js +2 -1
- package/lib/agents/media_mock_agent.js +24 -1
- package/lib/cli/commands/tool/scripting/builder.d.ts +1 -1
- package/lib/cli/commands/tool/story_to_script/builder.d.ts +1 -1
- package/lib/methods/mulmo_script.js +4 -0
- package/lib/utils/file.d.ts +2 -0
- package/lib/utils/file.js +5 -0
- package/lib/utils/provider2agent.d.ts +21 -0
- package/lib/utils/provider2agent.js +30 -3
- package/lib/utils/string.js +1 -0
- package/lib/utils/utils.d.ts +1 -1
- package/package.json +5 -4
- package/scripts/test/test_hello_bgm_0.json +21 -0
- package/scripts/test/test_hello_caption.json +38 -0
- package/scripts/test/test_hello_caption.json~ +21 -0
- package/scripts/test/test_hello_image.json +42 -0
- package/scripts/test/test_hello_image.json~ +18 -0
- package/scripts/test/test_hello_nobgm.json +21 -0
- package/scripts/test/test_replicate.json +21 -1
|
Binary file
|
package/lib/actions/audio.js
CHANGED
|
@@ -1,19 +1,14 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
2
|
import { GraphAI, TaskManager } from "graphai";
|
|
3
3
|
import * as agents from "@graphai/vanilla";
|
|
4
|
-
import ttsNijivoiceAgent from "../agents/tts_nijivoice_agent.js";
|
|
5
|
-
import addBGMAgent from "../agents/add_bgm_agent.js";
|
|
6
|
-
import combineAudioFilesAgent from "../agents/combine_audio_files_agent.js";
|
|
7
|
-
import ttsOpenaiAgent from "../agents/tts_openai_agent.js";
|
|
8
|
-
import ttsGoogleAgent from "../agents/tts_google_agent.js";
|
|
9
|
-
import ttsElevenlabsAgent from "../agents/tts_elevenlabs_agent.js";
|
|
10
4
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
11
|
-
import {
|
|
5
|
+
import { ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsElevenlabsAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent } from "../agents/index.js";
|
|
12
6
|
import { text2SpeechProviderSchema } from "../types/index.js";
|
|
13
7
|
import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
|
|
14
8
|
import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
|
|
15
9
|
import { text2hash, localizedText, settings2GraphAIConfig } from "../utils/utils.js";
|
|
16
10
|
import { provider2TTSAgent } from "../utils/provider2agent.js";
|
|
11
|
+
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
17
12
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
18
13
|
import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
|
|
19
14
|
const vanillaAgents = agents.default ?? agents;
|
|
@@ -85,6 +80,7 @@ const graph_tts = {
|
|
|
85
80
|
if: ":preprocessor.needsTTS",
|
|
86
81
|
agent: ":preprocessor.ttsAgent",
|
|
87
82
|
inputs: {
|
|
83
|
+
media: "audio",
|
|
88
84
|
text: ":preprocessor.text",
|
|
89
85
|
provider: ":preprocessor.provider",
|
|
90
86
|
lang: ":preprocessor.lang",
|
|
@@ -146,7 +142,6 @@ const graph_data = {
|
|
|
146
142
|
},
|
|
147
143
|
addBGM: {
|
|
148
144
|
agent: "addBGMAgent",
|
|
149
|
-
unless: ":context.presentationStyle.audioParams.bgmVolume.equal(0)",
|
|
150
145
|
inputs: {
|
|
151
146
|
wait: ":combineFiles",
|
|
152
147
|
voiceFile: ":audioCombinedFilePath",
|
|
@@ -156,18 +151,6 @@ const graph_data = {
|
|
|
156
151
|
musicFile: ":musicFile",
|
|
157
152
|
},
|
|
158
153
|
},
|
|
159
|
-
isResult: true,
|
|
160
|
-
defaultValue: {},
|
|
161
|
-
},
|
|
162
|
-
title: {
|
|
163
|
-
agent: "copyAgent",
|
|
164
|
-
params: {
|
|
165
|
-
namedKey: "title",
|
|
166
|
-
},
|
|
167
|
-
inputs: {
|
|
168
|
-
title: "\n${:context.studio.script.title}\n\n${:context.studio.script.description}\nReference: ${:context.studio.script.reference}\n",
|
|
169
|
-
waitFor: ":addBGM",
|
|
170
|
-
},
|
|
171
154
|
},
|
|
172
155
|
},
|
|
173
156
|
};
|
|
@@ -198,6 +181,7 @@ const audioAgents = {
|
|
|
198
181
|
ttsNijivoiceAgent,
|
|
199
182
|
ttsGoogleAgent,
|
|
200
183
|
ttsElevenlabsAgent,
|
|
184
|
+
mediaMockAgent,
|
|
201
185
|
addBGMAgent,
|
|
202
186
|
combineAudioFilesAgent,
|
|
203
187
|
};
|
|
@@ -4,7 +4,7 @@ import { getReferenceImagePath } from "../utils/file.js";
|
|
|
4
4
|
import { getExtention } from "../utils/utils.js";
|
|
5
5
|
import { graphOption } from "./images.js";
|
|
6
6
|
import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
|
|
7
|
-
import { imageGoogleAgent, imageOpenaiAgent } from "../agents/index.js";
|
|
7
|
+
import { imageGoogleAgent, imageOpenaiAgent, mediaMockAgent } from "../agents/index.js";
|
|
8
8
|
// public api
|
|
9
9
|
// Application may call this function directly to generate reference image.
|
|
10
10
|
export const generateReferenceImage = async (inputs) => {
|
|
@@ -21,6 +21,7 @@ export const generateReferenceImage = async (inputs) => {
|
|
|
21
21
|
agent: imageAgentInfo.agent,
|
|
22
22
|
retry: 2,
|
|
23
23
|
inputs: {
|
|
24
|
+
media: "image",
|
|
24
25
|
prompt,
|
|
25
26
|
cache: {
|
|
26
27
|
force: [context.force, force ?? false],
|
|
@@ -38,7 +39,7 @@ export const generateReferenceImage = async (inputs) => {
|
|
|
38
39
|
},
|
|
39
40
|
};
|
|
40
41
|
const options = await graphOption(context);
|
|
41
|
-
const graph = new GraphAI(image_graph_data, { imageGoogleAgent, imageOpenaiAgent }, options);
|
|
42
|
+
const graph = new GraphAI(image_graph_data, { imageGoogleAgent, imageOpenaiAgent, mediaMockAgent }, options);
|
|
42
43
|
await graph.run();
|
|
43
44
|
return imagePath;
|
|
44
45
|
};
|
package/lib/actions/images.js
CHANGED
|
@@ -22,6 +22,7 @@ const imageAgents = {
|
|
|
22
22
|
const movieAgents = {
|
|
23
23
|
movieGoogleAgent,
|
|
24
24
|
movieReplicateAgent,
|
|
25
|
+
mediaMockAgent,
|
|
25
26
|
};
|
|
26
27
|
const soundEffectAgents = {
|
|
27
28
|
soundEffectReplicateAgent,
|
|
@@ -77,6 +78,7 @@ const beat_graph_data = {
|
|
|
77
78
|
defaultValue: {},
|
|
78
79
|
agent: ":htmlImageAgentInfo.agent",
|
|
79
80
|
inputs: {
|
|
81
|
+
media: "html",
|
|
80
82
|
prompt: ":preprocessor.htmlPrompt",
|
|
81
83
|
system: ":preprocessor.htmlImageSystemPrompt",
|
|
82
84
|
params: {
|
|
@@ -122,6 +124,7 @@ const beat_graph_data = {
|
|
|
122
124
|
agent: ":preprocessor.imageAgentInfo.agent",
|
|
123
125
|
retry: 2,
|
|
124
126
|
inputs: {
|
|
127
|
+
media: "image",
|
|
125
128
|
prompt: ":preprocessor.prompt",
|
|
126
129
|
referenceImages: ":preprocessor.referenceImages",
|
|
127
130
|
cache: {
|
|
@@ -144,6 +147,7 @@ const beat_graph_data = {
|
|
|
144
147
|
if: ":preprocessor.movieFile",
|
|
145
148
|
agent: ":preprocessor.movieAgentInfo.agent",
|
|
146
149
|
inputs: {
|
|
150
|
+
media: "movie",
|
|
147
151
|
onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
|
|
148
152
|
prompt: ":beat.moviePrompt",
|
|
149
153
|
imagePath: ":preprocessor.referenceImageForMovie",
|
|
@@ -46,6 +46,10 @@ export declare const translateTextGraph: {
|
|
|
46
46
|
};
|
|
47
47
|
};
|
|
48
48
|
};
|
|
49
|
+
export declare const getOutputMultilingualFilePathAndMkdir: (context: MulmoStudioContext) => {
|
|
50
|
+
outputMultilingualFilePath: string;
|
|
51
|
+
outDirPath: string;
|
|
52
|
+
};
|
|
49
53
|
export declare const translateBeat: (index: number, context: MulmoStudioContext, targetLangs: string[], args?: {
|
|
50
54
|
settings?: Record<string, string>;
|
|
51
55
|
callbacks?: CallbackFunction[];
|
package/lib/actions/translate.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
|
-
import { createHash } from "crypto";
|
|
3
2
|
import fs from "fs";
|
|
4
3
|
import { GraphAI, assert, isNull, GraphAILogger } from "graphai";
|
|
5
4
|
import * as agents from "@graphai/vanilla";
|
|
@@ -9,13 +8,10 @@ import { splitText } from "../utils/string.js";
|
|
|
9
8
|
import { settings2GraphAIConfig } from "../utils/utils.js";
|
|
10
9
|
import { getMultiLingual } from "../utils/context.js";
|
|
11
10
|
import { currentMulmoScriptVersion } from "../utils/const.js";
|
|
12
|
-
import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
|
|
11
|
+
import { getOutputMultilingualFilePath, mkdir, writingMessage, hashSHA256 } from "../utils/file.js";
|
|
13
12
|
import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
|
|
14
13
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
15
14
|
const vanillaAgents = agents.default ?? agents;
|
|
16
|
-
const hashSHA256 = (text) => {
|
|
17
|
-
return createHash("sha256").update(text, "utf8").digest("hex");
|
|
18
|
-
};
|
|
19
15
|
// 1. translateGraph / map each beats.
|
|
20
16
|
// 2. beatGraph / map each target lang.
|
|
21
17
|
// 3. translateTextGraph / translate text.
|
|
@@ -202,6 +198,13 @@ const agentFilters = [
|
|
|
202
198
|
nodeIds: ["localizedText"],
|
|
203
199
|
},
|
|
204
200
|
];
|
|
201
|
+
export const getOutputMultilingualFilePathAndMkdir = (context) => {
|
|
202
|
+
const fileName = MulmoStudioContextMethods.getFileName(context);
|
|
203
|
+
const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
|
|
204
|
+
const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
|
|
205
|
+
mkdir(outDirPath);
|
|
206
|
+
return { outputMultilingualFilePath, outDirPath };
|
|
207
|
+
};
|
|
205
208
|
export const translateBeat = async (index, context, targetLangs, args) => {
|
|
206
209
|
const { settings, callbacks } = args ?? {};
|
|
207
210
|
// Validate inputs
|
|
@@ -212,10 +215,7 @@ export const translateBeat = async (index, context, targetLangs, args) => {
|
|
|
212
215
|
throw new Error("targetLangs must be a non-empty array");
|
|
213
216
|
}
|
|
214
217
|
try {
|
|
215
|
-
const
|
|
216
|
-
const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
|
|
217
|
-
const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
|
|
218
|
-
mkdir(outDirPath);
|
|
218
|
+
const { outputMultilingualFilePath } = getOutputMultilingualFilePathAndMkdir(context);
|
|
219
219
|
const config = settings2GraphAIConfig(settings, process.env);
|
|
220
220
|
assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
|
|
221
221
|
const graph = new GraphAI(beatGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
|
|
@@ -246,10 +246,7 @@ export const translate = async (context, args) => {
|
|
|
246
246
|
const { settings, callbacks } = args ?? {};
|
|
247
247
|
try {
|
|
248
248
|
MulmoStudioContextMethods.setSessionState(context, "multiLingual", true);
|
|
249
|
-
const
|
|
250
|
-
const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
|
|
251
|
-
const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
|
|
252
|
-
mkdir(outDirPath);
|
|
249
|
+
const { outputMultilingualFilePath, outDirPath } = getOutputMultilingualFilePathAndMkdir(context);
|
|
253
250
|
const targetLangs = [...new Set([context.lang, context.studio.script.captionParams?.lang].filter((x) => !isNull(x)))];
|
|
254
251
|
const config = settings2GraphAIConfig(settings, process.env);
|
|
255
252
|
assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
|
package/lib/agents/index.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ import mediaMockAgent from "./media_mock_agent.js";
|
|
|
9
9
|
import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
|
|
10
10
|
import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
|
|
11
11
|
import ttsOpenaiAgent from "./tts_openai_agent.js";
|
|
12
|
+
import ttsGoogleAgent from "./tts_google_agent.js";
|
|
12
13
|
import validateSchemaAgent from "./validate_schema_agent.js";
|
|
13
14
|
import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
|
|
14
15
|
import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
|
|
@@ -16,4 +17,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
|
|
|
16
17
|
import { textInputAgent } from "@graphai/input_agents";
|
|
17
18
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
18
19
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
19
|
-
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
|
|
20
|
+
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
|
package/lib/agents/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import mediaMockAgent from "./media_mock_agent.js";
|
|
|
9
9
|
import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
|
|
10
10
|
import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
|
|
11
11
|
import ttsOpenaiAgent from "./tts_openai_agent.js";
|
|
12
|
+
import ttsGoogleAgent from "./tts_google_agent.js";
|
|
12
13
|
import validateSchemaAgent from "./validate_schema_agent.js";
|
|
13
14
|
import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
|
|
14
15
|
import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
|
|
@@ -17,4 +18,4 @@ import { textInputAgent } from "@graphai/input_agents";
|
|
|
17
18
|
import { openAIAgent } from "@graphai/openai_agent";
|
|
18
19
|
// import * as vanilla from "@graphai/vanilla";
|
|
19
20
|
import { fileWriteAgent } from "@graphai/vanilla_node_agents";
|
|
20
|
-
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
|
|
21
|
+
export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
|
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
|
-
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import { silent60secPath, mulmoCreditPath } from "../utils/file.js";
|
|
4
|
+
export const mediaMockAgent = async ({ namedInputs }) => {
|
|
5
|
+
if (namedInputs.media === "audio") {
|
|
6
|
+
const buffer = fs.readFileSync(silent60secPath());
|
|
7
|
+
return { buffer };
|
|
8
|
+
}
|
|
9
|
+
if (namedInputs.media === "image") {
|
|
10
|
+
const buffer = fs.readFileSync(mulmoCreditPath());
|
|
11
|
+
return { buffer };
|
|
12
|
+
}
|
|
13
|
+
if (namedInputs.media === "movie") {
|
|
14
|
+
const url = "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.mov";
|
|
15
|
+
const res = await fetch(url);
|
|
16
|
+
if (!res.ok) {
|
|
17
|
+
throw new Error(`Failed to fetch: ${res.status} ${res.statusText}`);
|
|
18
|
+
}
|
|
19
|
+
const arrayBuffer = await res.arrayBuffer();
|
|
20
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
21
|
+
return { buffer };
|
|
22
|
+
}
|
|
23
|
+
if (namedInputs.media === "html") {
|
|
24
|
+
return { text: "<html><title>test</title><body>test</body></html>" };
|
|
25
|
+
}
|
|
3
26
|
GraphAILogger.debug("agent dryRun");
|
|
4
27
|
return { buffer: Buffer.from([]) };
|
|
5
28
|
};
|
|
@@ -16,7 +16,7 @@ export declare const builder: (yargs: Argv) => Argv<{
|
|
|
16
16
|
} & {
|
|
17
17
|
s: string;
|
|
18
18
|
} & {
|
|
19
|
-
llm: "openai" | "anthropic" | "gemini" | "groq" | undefined;
|
|
19
|
+
llm: "mock" | "openai" | "anthropic" | "gemini" | "groq" | undefined;
|
|
20
20
|
} & {
|
|
21
21
|
llm_model: string | undefined;
|
|
22
22
|
}>;
|
|
@@ -10,7 +10,7 @@ export declare const builder: (yargs: Argv) => Argv<{
|
|
|
10
10
|
} & {
|
|
11
11
|
beats_per_scene: number;
|
|
12
12
|
} & {
|
|
13
|
-
llm: "openai" | "anthropic" | "gemini" | "groq" | undefined;
|
|
13
|
+
llm: "mock" | "openai" | "anthropic" | "gemini" | "groq" | undefined;
|
|
14
14
|
} & {
|
|
15
15
|
llm_model: string | undefined;
|
|
16
16
|
} & {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { GraphAILogger } from "graphai";
|
|
1
2
|
import { mulmoScriptSchema, mulmoStudioMultiLingualFileSchema } from "../types/index.js";
|
|
2
3
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
3
4
|
const validate_1_0 = (script) => {
|
|
@@ -37,6 +38,9 @@ export const MulmoStudioMultiLingualMethod = {
|
|
|
37
38
|
validate(jsonData, studioBeatsLength) {
|
|
38
39
|
// TODO version check
|
|
39
40
|
const result = mulmoStudioMultiLingualFileSchema.safeParse(jsonData);
|
|
41
|
+
if (!result.success) {
|
|
42
|
+
GraphAILogger.warn("multiLingual file validation failed.");
|
|
43
|
+
}
|
|
40
44
|
const dataSet = result.success ? result.data.multiLingual : [];
|
|
41
45
|
while (dataSet.length < studioBeatsLength) {
|
|
42
46
|
dataSet.push({ multiLingualTexts: {} });
|
package/lib/utils/file.d.ts
CHANGED
|
@@ -37,6 +37,7 @@ export declare const getPromptTemplateFilePath: (promptTemplateName: string) =>
|
|
|
37
37
|
export declare const mkdir: (dirPath: string) => void;
|
|
38
38
|
export declare const silent60secPath: () => string;
|
|
39
39
|
export declare const defaultBGMPath: () => string;
|
|
40
|
+
export declare const mulmoCreditPath: () => string;
|
|
40
41
|
export declare const getHTMLFile: (filename: string) => string;
|
|
41
42
|
export declare const getBaseDirPath: (basedir?: string) => string;
|
|
42
43
|
export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
|
|
@@ -48,3 +49,4 @@ export declare const getAvailableScriptTemplates: () => MulmoScript[];
|
|
|
48
49
|
export declare const writingMessage: (filePath: string) => void;
|
|
49
50
|
export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;
|
|
50
51
|
export declare const generateTimestampedFileName: (prefix: string) => string;
|
|
52
|
+
export declare const hashSHA256: (text: string) => string;
|
package/lib/utils/file.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
|
+
import { createHash } from "crypto";
|
|
3
4
|
import { parse as yamlParse } from "yaml";
|
|
4
5
|
import { fileURLToPath } from "url";
|
|
5
6
|
import { GraphAILogger } from "graphai";
|
|
@@ -131,6 +132,7 @@ export const mkdir = (dirPath) => {
|
|
|
131
132
|
// export const silentLastPath = path.resolve(npmRoot, "./assets/audio/silent800.mp3");
|
|
132
133
|
export const silent60secPath = () => path.resolve(npmRoot, "./assets/audio/silent60sec.mp3");
|
|
133
134
|
export const defaultBGMPath = () => "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story002.mp3";
|
|
135
|
+
export const mulmoCreditPath = () => path.resolve(npmRoot, "./assets/images/mulmocast_credit.png");
|
|
134
136
|
export const getHTMLFile = (filename) => {
|
|
135
137
|
const htmlPath = path.resolve(npmRoot, `./assets/html/${filename}.html`);
|
|
136
138
|
return fs.readFileSync(htmlPath, "utf-8");
|
|
@@ -226,3 +228,6 @@ export const generateTimestampedFileName = (prefix) => {
|
|
|
226
228
|
const pad = (n) => n.toString().padStart(2, "0");
|
|
227
229
|
return `${prefix}_${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
|
228
230
|
};
|
|
231
|
+
export const hashSHA256 = (text) => {
|
|
232
|
+
return createHash("sha256").update(text, "utf8").digest("hex");
|
|
233
|
+
};
|
|
@@ -19,6 +19,12 @@ export declare const provider2TTSAgent: {
|
|
|
19
19
|
defaultModel: string;
|
|
20
20
|
models: string[];
|
|
21
21
|
};
|
|
22
|
+
mock: {
|
|
23
|
+
agentName: string;
|
|
24
|
+
hasLimitedConcurrency: boolean;
|
|
25
|
+
defaultModel: string;
|
|
26
|
+
models: string[];
|
|
27
|
+
};
|
|
22
28
|
};
|
|
23
29
|
export declare const provider2ImageAgent: {
|
|
24
30
|
openai: {
|
|
@@ -31,6 +37,11 @@ export declare const provider2ImageAgent: {
|
|
|
31
37
|
defaultModel: string;
|
|
32
38
|
models: string[];
|
|
33
39
|
};
|
|
40
|
+
mock: {
|
|
41
|
+
agentName: string;
|
|
42
|
+
defaultModel: string;
|
|
43
|
+
models: string[];
|
|
44
|
+
};
|
|
34
45
|
};
|
|
35
46
|
export type ReplicateModel = `${string}/${string}`;
|
|
36
47
|
export declare const provider2MovieAgent: {
|
|
@@ -50,6 +61,11 @@ export declare const provider2MovieAgent: {
|
|
|
50
61
|
defaultModel: string;
|
|
51
62
|
models: string[];
|
|
52
63
|
};
|
|
64
|
+
mock: {
|
|
65
|
+
agentName: string;
|
|
66
|
+
defaultModel: string;
|
|
67
|
+
models: string[];
|
|
68
|
+
};
|
|
53
69
|
};
|
|
54
70
|
export declare const provider2SoundEffectAgent: {
|
|
55
71
|
replicate: {
|
|
@@ -95,6 +111,11 @@ export declare const provider2LLMAgent: {
|
|
|
95
111
|
readonly defaultModel: "llama3-8b-8192";
|
|
96
112
|
readonly max_tokens: 4096;
|
|
97
113
|
};
|
|
114
|
+
readonly mock: {
|
|
115
|
+
readonly agentName: "mediaMockAgent";
|
|
116
|
+
readonly defaultModel: "mock";
|
|
117
|
+
readonly max_tokens: 4096;
|
|
118
|
+
};
|
|
98
119
|
};
|
|
99
120
|
export declare const defaultProviders: {
|
|
100
121
|
tts: keyof typeof provider2TTSAgent;
|
|
@@ -22,6 +22,12 @@ export const provider2TTSAgent = {
|
|
|
22
22
|
// https://elevenlabs.io/docs/models
|
|
23
23
|
models: ["eleven_multilingual_v2", "eleven_turbo_v2_5", "eleven_turbo_v2", "eleven_flash_v2_5", "eleven_flash_v2"],
|
|
24
24
|
},
|
|
25
|
+
mock: {
|
|
26
|
+
agentName: "mediaMockAgent",
|
|
27
|
+
hasLimitedConcurrency: true,
|
|
28
|
+
defaultModel: "mock-model",
|
|
29
|
+
models: ["mock-model"],
|
|
30
|
+
},
|
|
25
31
|
};
|
|
26
32
|
export const provider2ImageAgent = {
|
|
27
33
|
openai: {
|
|
@@ -34,6 +40,11 @@ export const provider2ImageAgent = {
|
|
|
34
40
|
defaultModel: "imagen-3.0-fast-generate-001",
|
|
35
41
|
models: ["imagen-3.0-fast-generate-001", "imagen-3.0-generate-002", "imagen-3.0-capability-001"],
|
|
36
42
|
},
|
|
43
|
+
mock: {
|
|
44
|
+
agentName: "mediaMockAgent",
|
|
45
|
+
defaultModel: "mock-model",
|
|
46
|
+
models: ["mock-model"],
|
|
47
|
+
},
|
|
37
48
|
};
|
|
38
49
|
export const provider2MovieAgent = {
|
|
39
50
|
replicate: {
|
|
@@ -50,6 +61,7 @@ export const provider2MovieAgent = {
|
|
|
50
61
|
"google/veo-3-fast",
|
|
51
62
|
"minimax/video-01",
|
|
52
63
|
"minimax/hailuo-02",
|
|
64
|
+
"minimax/hailuo-02-fast",
|
|
53
65
|
"pixverse/pixverse-v4.5",
|
|
54
66
|
"wan-video/wan-2.2-i2v-480p-fast",
|
|
55
67
|
"wan-video/wan-2.2-t2v-480p-fast",
|
|
@@ -89,12 +101,12 @@ export const provider2MovieAgent = {
|
|
|
89
101
|
},
|
|
90
102
|
"google/veo-3": {
|
|
91
103
|
durations: [8],
|
|
92
|
-
start_image:
|
|
104
|
+
start_image: "image",
|
|
93
105
|
price_per_sec: 0.75,
|
|
94
106
|
},
|
|
95
107
|
"google/veo-3-fast": {
|
|
96
108
|
durations: [8],
|
|
97
|
-
start_image:
|
|
109
|
+
start_image: "image",
|
|
98
110
|
price_per_sec: 0.4,
|
|
99
111
|
},
|
|
100
112
|
"minimax/video-01": {
|
|
@@ -107,6 +119,11 @@ export const provider2MovieAgent = {
|
|
|
107
119
|
start_image: "first_frame_image",
|
|
108
120
|
price_per_sec: 0.08,
|
|
109
121
|
},
|
|
122
|
+
"minimax/hailuo-02-fast": {
|
|
123
|
+
durations: [6, 10], // NOTE: 512P
|
|
124
|
+
start_image: "first_frame_image",
|
|
125
|
+
price_per_sec: 0.0166,
|
|
126
|
+
},
|
|
110
127
|
"pixverse/pixverse-v4.5": {
|
|
111
128
|
durations: [5, 8],
|
|
112
129
|
start_image: "image",
|
|
@@ -130,6 +147,11 @@ export const provider2MovieAgent = {
|
|
|
130
147
|
defaultModel: "veo-2.0-generate-001",
|
|
131
148
|
models: ["veo-2.0-generate-001"],
|
|
132
149
|
},
|
|
150
|
+
mock: {
|
|
151
|
+
agentName: "mediaMockAgent",
|
|
152
|
+
defaultModel: "mock-model",
|
|
153
|
+
models: ["mock-model"],
|
|
154
|
+
},
|
|
133
155
|
};
|
|
134
156
|
export const provider2SoundEffectAgent = {
|
|
135
157
|
replicate: {
|
|
@@ -202,6 +224,11 @@ export const provider2LLMAgent = {
|
|
|
202
224
|
defaultModel: "llama3-8b-8192",
|
|
203
225
|
max_tokens: 4096,
|
|
204
226
|
},
|
|
227
|
+
mock: {
|
|
228
|
+
agentName: "mediaMockAgent",
|
|
229
|
+
defaultModel: "mock",
|
|
230
|
+
max_tokens: 4096,
|
|
231
|
+
},
|
|
205
232
|
};
|
|
206
233
|
export const defaultProviders = {
|
|
207
234
|
tts: "openai",
|
|
@@ -213,4 +240,4 @@ export const defaultProviders = {
|
|
|
213
240
|
lipSync: "replicate",
|
|
214
241
|
};
|
|
215
242
|
export const llm = Object.keys(provider2LLMAgent);
|
|
216
|
-
export const htmlLLMProvider = ["openai", "anthropic"];
|
|
243
|
+
export const htmlLLMProvider = ["openai", "anthropic", "mock"];
|
package/lib/utils/string.js
CHANGED
|
@@ -46,6 +46,7 @@ export const replacementsJa = [
|
|
|
46
46
|
{ from: "Groq", to: "グロック" },
|
|
47
47
|
{ from: "TSMC", to: "ティーエスエムシー" },
|
|
48
48
|
{ from: "NVIDIA", to: "エヌビディア" },
|
|
49
|
+
{ from: "PER", to: "ピーイーアール" },
|
|
49
50
|
{ from: "1つ", to: "ひとつ" },
|
|
50
51
|
{ from: "2つ", to: "ふたつ" },
|
|
51
52
|
{ from: "3つ", to: "みっつ" },
|
package/lib/utils/utils.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { ConfigDataDictionary, DefaultConfigData } from "graphai";
|
|
|
2
2
|
import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
|
|
3
3
|
import type { LLM } from "./provider2agent.js";
|
|
4
4
|
export declare const llmPair: (_llm?: LLM, _model?: string) => {
|
|
5
|
-
agent: "openAIAgent" | "anthropicAgent" | "geminiAgent" | "groqAgent";
|
|
5
|
+
agent: "mediaMockAgent" | "openAIAgent" | "anthropicAgent" | "geminiAgent" | "groqAgent";
|
|
6
6
|
model: string;
|
|
7
7
|
max_tokens: 8192 | 4096;
|
|
8
8
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.11",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.node.js",
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"./scripts/test",
|
|
27
27
|
"./assets/audio/silent60sec.mp3",
|
|
28
28
|
"./assets/html/",
|
|
29
|
+
"./assets/images/",
|
|
29
30
|
"./assets/templates/"
|
|
30
31
|
],
|
|
31
32
|
"directories": {
|
|
@@ -82,8 +83,8 @@
|
|
|
82
83
|
"fluent-ffmpeg": "^2.1.3",
|
|
83
84
|
"google-auth-library": "^10.1.0",
|
|
84
85
|
"graphai": "^2.0.13",
|
|
85
|
-
"inquirer": "^12.
|
|
86
|
-
"marked": "^16.1.
|
|
86
|
+
"inquirer": "^12.9.0",
|
|
87
|
+
"marked": "^16.1.2",
|
|
87
88
|
"ora": "^8.2.0",
|
|
88
89
|
"puppeteer": "^24.15.0",
|
|
89
90
|
"replicate": "^1.0.1",
|
|
@@ -106,7 +107,7 @@
|
|
|
106
107
|
"ts-node": "^10.9.2",
|
|
107
108
|
"tsx": "^4.20.3",
|
|
108
109
|
"typescript": "^5.9.2",
|
|
109
|
-
"typescript-eslint": "^8.
|
|
110
|
+
"typescript-eslint": "^8.39.0"
|
|
110
111
|
},
|
|
111
112
|
"engines": {
|
|
112
113
|
"node": ">=18.0.0"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"audioParams": {
|
|
6
|
+
"bgmVolume": 0
|
|
7
|
+
},
|
|
8
|
+
"lang": "en",
|
|
9
|
+
"beats": [
|
|
10
|
+
{
|
|
11
|
+
"text": "Hello World",
|
|
12
|
+
"image": {
|
|
13
|
+
"type": "textSlide",
|
|
14
|
+
"slide": {
|
|
15
|
+
"title": "Hello World",
|
|
16
|
+
"bullets": ["Hello", "World"]
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
]
|
|
21
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"captionParams": {
|
|
7
|
+
"lang": "en"
|
|
8
|
+
},
|
|
9
|
+
"movieParams": {
|
|
10
|
+
"provider": "mock"
|
|
11
|
+
},
|
|
12
|
+
"speechParams": {
|
|
13
|
+
"speakers": {
|
|
14
|
+
"Presenter": {
|
|
15
|
+
"displayName": {
|
|
16
|
+
"en": "Presenter"
|
|
17
|
+
},
|
|
18
|
+
"voiceId": "ae42d8b0-0c4b-4289-95b7-cf988569af36",
|
|
19
|
+
"provider": "mock"
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"imageParams": {
|
|
24
|
+
"provider": "mock"
|
|
25
|
+
},
|
|
26
|
+
"beats": [
|
|
27
|
+
{
|
|
28
|
+
"text": "Hello World",
|
|
29
|
+
"image": {
|
|
30
|
+
"type": "textSlide",
|
|
31
|
+
"slide": {
|
|
32
|
+
"title": "Hello World",
|
|
33
|
+
"bullets": ["Hello", "World"]
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"captionParams": {
|
|
7
|
+
"lang": "en"
|
|
8
|
+
},
|
|
9
|
+
"beats": [
|
|
10
|
+
{
|
|
11
|
+
"text": "Hello World",
|
|
12
|
+
"image": {
|
|
13
|
+
"type": "textSlide",
|
|
14
|
+
"slide": {
|
|
15
|
+
"title": "Hello World",
|
|
16
|
+
"bullets": ["Hello", "World"]
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
]
|
|
21
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"movieParams": {
|
|
7
|
+
"provider": "mock"
|
|
8
|
+
},
|
|
9
|
+
"speechParams": {
|
|
10
|
+
"speakers": {
|
|
11
|
+
"Presenter": {
|
|
12
|
+
"displayName": {
|
|
13
|
+
"en": "Presenter"
|
|
14
|
+
},
|
|
15
|
+
"voiceId": "ae42d8b0-0c4b-4289-95b7-cf988569af36",
|
|
16
|
+
"provider": "mock"
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"htmlImageParams": {
|
|
21
|
+
"provider": "mock"
|
|
22
|
+
},
|
|
23
|
+
"imageParams": {
|
|
24
|
+
"provider": "mock"
|
|
25
|
+
},
|
|
26
|
+
"beats": [
|
|
27
|
+
{
|
|
28
|
+
"text": "Hello World"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"text": "Hello World",
|
|
32
|
+
"moviePrompt": "fire"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"text": "Hello World",
|
|
36
|
+
"htmlPrompt": {
|
|
37
|
+
"prompt": "fire",
|
|
38
|
+
"data": []
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$mulmocast": {
|
|
3
|
+
"version": "1.1"
|
|
4
|
+
},
|
|
5
|
+
"lang": "en",
|
|
6
|
+
"audioParams": {
|
|
7
|
+
"bgmVolume": 0
|
|
8
|
+
},
|
|
9
|
+
"beats": [
|
|
10
|
+
{
|
|
11
|
+
"text": "Hello World",
|
|
12
|
+
"image": {
|
|
13
|
+
"type": "textSlide",
|
|
14
|
+
"slide": {
|
|
15
|
+
"title": "Hello World",
|
|
16
|
+
"bullets": ["Hello", "World"]
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
]
|
|
21
|
+
}
|
|
@@ -48,6 +48,16 @@
|
|
|
48
48
|
"model": "google/veo-3-fast"
|
|
49
49
|
}
|
|
50
50
|
},
|
|
51
|
+
{
|
|
52
|
+
"id": "veo-3-fast with a start image",
|
|
53
|
+
"text": "google/veo-3-fast with a start image",
|
|
54
|
+
"duration": 8,
|
|
55
|
+
"imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
56
|
+
"moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
57
|
+
"movieParams": {
|
|
58
|
+
"model": "google/veo-3-fast"
|
|
59
|
+
}
|
|
60
|
+
},
|
|
51
61
|
{
|
|
52
62
|
"id": "seedance-1-lite",
|
|
53
63
|
"text": "bytedance seedance-1-lite",
|
|
@@ -114,7 +124,7 @@
|
|
|
114
124
|
},
|
|
115
125
|
{
|
|
116
126
|
"id": "hailuo-02-image",
|
|
117
|
-
"text": "minimax/hailuo-02 with
|
|
127
|
+
"text": "minimax/hailuo-02 with start image",
|
|
118
128
|
"duration": 6,
|
|
119
129
|
"imagePrompt": "a cat is doing an acrobatic dive into a swimming pool at the olympics, from a 10m high diving board, Ghibli style",
|
|
120
130
|
"moviePrompt": "televised footage of a cat is doing an acrobatic dive into a swimming pool at the olympics, from a 10m high diving board, flips and spins",
|
|
@@ -122,6 +132,16 @@
|
|
|
122
132
|
"model": "minimax/hailuo-02"
|
|
123
133
|
}
|
|
124
134
|
},
|
|
135
|
+
{
|
|
136
|
+
"id": "hailuo-02-fast",
|
|
137
|
+
"text": "minimax/hailuo-02-fast with no start image",
|
|
138
|
+
"duration": 10,
|
|
139
|
+
"imagePrompt": "televised footage of a cat preparing to dive into a swimming pool at the olympics, from a 10m high diving board",
|
|
140
|
+
"moviePrompt": "a cat is doing an acrobatic dive into a swimming pool at the olympics, from a 10m high diving board, flips and spins",
|
|
141
|
+
"movieParams": {
|
|
142
|
+
"model": "minimax/hailuo-02-fast"
|
|
143
|
+
}
|
|
144
|
+
},
|
|
125
145
|
{
|
|
126
146
|
"id": "wan-2.2-i2v-480p-fast",
|
|
127
147
|
"text": "wan-video/wan-2.2-i2v-480p-fast",
|