mulmocast 2.0.8 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/actions/movie.d.ts +29 -2
- package/lib/actions/movie.js +243 -91
- package/lib/methods/mulmo_presentation_style.d.ts +4 -3
- package/lib/methods/mulmo_presentation_style.js +7 -1
- package/lib/types/schema.d.ts +202 -31
- package/lib/types/schema.js +27 -13
- package/lib/types/schema_video_filter.d.ts +423 -0
- package/lib/types/schema_video_filter.js +253 -0
- package/lib/types/type.d.ts +2 -1
- package/lib/utils/context.d.ts +11 -3
- package/lib/utils/ffmpeg_utils.js +2 -2
- package/lib/utils/provider2agent.d.ts +4 -0
- package/lib/utils/provider2agent.js +5 -0
- package/lib/utils/utils.js +24 -55
- package/lib/utils/video_filter.d.ts +7 -0
- package/lib/utils/video_filter.js +149 -0
- package/package.json +7 -7
- package/scripts/test/README.md +48 -48
- package/scripts/test/test_transition2.json +460 -0
- package/scripts/test/test_transition2.json~ +62 -0
- package/scripts/test/test_transition3.json +70 -0
- package/scripts/test/test_transition3.json~ +76 -0
- package/scripts/test/test_transition_no_audio.json +16 -0
- package/scripts/test/test_video_filters.json~ +227 -0
- package/scripts/test/test_wipe_simple.json +37 -0
- package/scripts/test/test_all_image.json~ +0 -45
- package/scripts/test/test_all_movie.json~ +0 -37
- package/scripts/test/test_all_tts.json~ +0 -83
- package/scripts/test/test_audio_gemini.json~ +0 -67
- package/scripts/test/test_genai2.json~ +0 -84
- package/scripts/test/test_genai_movie.json~ +0 -22
- package/scripts/test/test_kotodama.json~ +0 -0
- package/scripts/test/test_lipsync2.json~ +0 -24
- package/scripts/test/test_movie2.json~ +0 -40
- package/scripts/test/test_play_to_end.json~ +0 -65
package/lib/types/type.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type CallbackFunction } from "graphai";
|
|
2
|
-
import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
|
|
2
|
+
import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
|
|
3
3
|
import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
|
|
4
4
|
import type { LLM } from "../utils/provider2agent.js";
|
|
5
5
|
import { z } from "zod";
|
|
@@ -14,6 +14,7 @@ export type MulmoImageParams = z.infer<typeof mulmoImageParamsSchema>;
|
|
|
14
14
|
export type MulmoImageParamsImagesValue = z.infer<typeof mulmoImageParamsImagesValueSchema>;
|
|
15
15
|
export type MulmoImageParamsImages = z.infer<typeof mulmoImageParamsImagesSchema>;
|
|
16
16
|
export type MulmoFillOption = z.infer<typeof mulmoFillOptionSchema>;
|
|
17
|
+
export type MulmoTransition = z.infer<typeof mulmoTransitionSchema>;
|
|
17
18
|
export type TextSlideParams = z.infer<typeof textSlideParamsSchema>;
|
|
18
19
|
export type Text2ImageProvider = z.infer<typeof text2ImageProviderSchema>;
|
|
19
20
|
export type Text2HtmlImageProvider = z.infer<typeof text2HtmlImageProviderSchema>;
|
package/lib/utils/context.d.ts
CHANGED
|
@@ -65,7 +65,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
65
65
|
style: "aspectFit" | "aspectFill";
|
|
66
66
|
} | undefined;
|
|
67
67
|
transition?: {
|
|
68
|
-
type: "fade" | "slideout_left";
|
|
68
|
+
type: "fade" | "slideout_left" | "slideout_right" | "slideout_up" | "slideout_down" | "slidein_left" | "slidein_right" | "slidein_up" | "slidein_down" | "wipeleft" | "wiperight" | "wipeup" | "wipedown" | "wipetl" | "wipetr" | "wipebl" | "wipebr";
|
|
69
69
|
duration: number;
|
|
70
70
|
} | undefined;
|
|
71
71
|
};
|
|
@@ -228,6 +228,10 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
|
|
|
228
228
|
fillOption?: {
|
|
229
229
|
style: "aspectFit" | "aspectFill";
|
|
230
230
|
} | undefined;
|
|
231
|
+
transition?: {
|
|
232
|
+
type: "fade" | "slideout_left" | "slideout_right" | "slideout_up" | "slideout_down" | "slidein_left" | "slidein_right" | "slidein_up" | "slidein_down" | "wipeleft" | "wiperight" | "wipeup" | "wipedown" | "wipetl" | "wipetr" | "wipebl" | "wipebr";
|
|
233
|
+
duration: number;
|
|
234
|
+
} | undefined;
|
|
231
235
|
speed?: number | undefined;
|
|
232
236
|
} | undefined;
|
|
233
237
|
soundEffectParams?: {
|
|
@@ -383,7 +387,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
383
387
|
style: "aspectFit" | "aspectFill";
|
|
384
388
|
} | undefined;
|
|
385
389
|
transition?: {
|
|
386
|
-
type: "fade" | "slideout_left";
|
|
390
|
+
type: "fade" | "slideout_left" | "slideout_right" | "slideout_up" | "slideout_down" | "slidein_left" | "slidein_right" | "slidein_up" | "slidein_down" | "wipeleft" | "wiperight" | "wipeup" | "wipedown" | "wipetl" | "wipetr" | "wipebl" | "wipebr";
|
|
387
391
|
duration: number;
|
|
388
392
|
} | undefined;
|
|
389
393
|
};
|
|
@@ -546,6 +550,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
546
550
|
fillOption?: {
|
|
547
551
|
style: "aspectFit" | "aspectFill";
|
|
548
552
|
} | undefined;
|
|
553
|
+
transition?: {
|
|
554
|
+
type: "fade" | "slideout_left" | "slideout_right" | "slideout_up" | "slideout_down" | "slidein_left" | "slidein_right" | "slidein_up" | "slidein_down" | "wipeleft" | "wiperight" | "wipeup" | "wipedown" | "wipetl" | "wipetr" | "wipebl" | "wipebr";
|
|
555
|
+
duration: number;
|
|
556
|
+
} | undefined;
|
|
549
557
|
speed?: number | undefined;
|
|
550
558
|
} | undefined;
|
|
551
559
|
soundEffectParams?: {
|
|
@@ -708,7 +716,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
|
|
|
708
716
|
style: "aspectFit" | "aspectFill";
|
|
709
717
|
} | undefined;
|
|
710
718
|
transition?: {
|
|
711
|
-
type: "fade" | "slideout_left";
|
|
719
|
+
type: "fade" | "slideout_left" | "slideout_right" | "slideout_up" | "slideout_down" | "slidein_left" | "slidein_right" | "slidein_up" | "slidein_down" | "wipeleft" | "wiperight" | "wipeup" | "wipedown" | "wipetl" | "wipetr" | "wipebl" | "wipebr";
|
|
712
720
|
duration: number;
|
|
713
721
|
} | undefined;
|
|
714
722
|
};
|
|
@@ -46,8 +46,8 @@ export const FfmpegContextGenerateOutput = (context, output, options = []) => {
|
|
|
46
46
|
.complexFilter(context.filterComplex)
|
|
47
47
|
.outputOptions(options)
|
|
48
48
|
.output(output)
|
|
49
|
-
.on("start", (
|
|
50
|
-
GraphAILogger.log("Started FFmpeg
|
|
49
|
+
.on("start", (cmdLine) => {
|
|
50
|
+
GraphAILogger.log("Started FFmpeg with command:", cmdLine);
|
|
51
51
|
})
|
|
52
52
|
.on("error", (err, stdout, stderr) => {
|
|
53
53
|
GraphAILogger.error("Error occurred:", err);
|
|
@@ -10,6 +10,7 @@ export declare const provider2TTSAgent: {
|
|
|
10
10
|
defaultModel: string;
|
|
11
11
|
defaultVoice: string;
|
|
12
12
|
keyName: string;
|
|
13
|
+
baseURLKeyName: string;
|
|
13
14
|
};
|
|
14
15
|
google: {
|
|
15
16
|
agentName: string;
|
|
@@ -49,6 +50,7 @@ export declare const provider2ImageAgent: {
|
|
|
49
50
|
defaultModel: string;
|
|
50
51
|
models: string[];
|
|
51
52
|
keyName: string;
|
|
53
|
+
baseURLKeyName: string;
|
|
52
54
|
};
|
|
53
55
|
google: {
|
|
54
56
|
agentName: string;
|
|
@@ -138,6 +140,7 @@ export declare const provider2LLMAgent: {
|
|
|
138
140
|
readonly agentName: "openAIAgent";
|
|
139
141
|
readonly defaultModel: "gpt-5";
|
|
140
142
|
readonly keyName: "OPENAI_API_KEY";
|
|
143
|
+
readonly baseURLKeyName: "OPENAI_BASE_URL";
|
|
141
144
|
readonly max_tokens: 8192;
|
|
142
145
|
readonly models: readonly ["gpt-5", "gpt-5-nano", "gpt-5-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", "o3", "o3-mini", "o3-pro", "o1", "o1-pro", "gpt-4o", "gpt-4o-mini"];
|
|
143
146
|
};
|
|
@@ -147,6 +150,7 @@ export declare const provider2LLMAgent: {
|
|
|
147
150
|
readonly max_tokens: 8192;
|
|
148
151
|
readonly models: readonly ["claude-opus-4-1-20250805", "claude-opus-4-20250514", "claude-sonnet-4-20250514", "claude-3-7-sonnet-20250219", "claude-3-haiku-20240307"];
|
|
149
152
|
readonly keyName: "ANTHROPIC_API_KEY";
|
|
153
|
+
readonly apiKeyNameOverride: "ANTHROPIC_API_TOKEN";
|
|
150
154
|
};
|
|
151
155
|
readonly gemini: {
|
|
152
156
|
readonly agentName: "geminiAgent";
|
|
@@ -11,6 +11,7 @@ export const provider2TTSAgent = {
|
|
|
11
11
|
defaultModel: "gpt-4o-mini-tts",
|
|
12
12
|
defaultVoice: "shimmer",
|
|
13
13
|
keyName: "OPENAI_API_KEY",
|
|
14
|
+
baseURLKeyName: "OPENAI_BASE_URL",
|
|
14
15
|
},
|
|
15
16
|
google: {
|
|
16
17
|
agentName: "ttsGoogleAgent",
|
|
@@ -52,6 +53,7 @@ export const provider2ImageAgent = {
|
|
|
52
53
|
defaultModel: "gpt-image-1",
|
|
53
54
|
models: ["dall-e-3", "gpt-image-1"],
|
|
54
55
|
keyName: "OPENAI_API_KEY",
|
|
56
|
+
baseURLKeyName: "OPENAI_BASE_URL",
|
|
55
57
|
},
|
|
56
58
|
google: {
|
|
57
59
|
agentName: "imageGenAIAgent",
|
|
@@ -251,6 +253,7 @@ export const provider2LLMAgent = {
|
|
|
251
253
|
agentName: "openAIAgent",
|
|
252
254
|
defaultModel: "gpt-5",
|
|
253
255
|
keyName: "OPENAI_API_KEY",
|
|
256
|
+
baseURLKeyName: "OPENAI_BASE_URL",
|
|
254
257
|
max_tokens: 8192,
|
|
255
258
|
models: [
|
|
256
259
|
"gpt-5",
|
|
@@ -274,6 +277,8 @@ export const provider2LLMAgent = {
|
|
|
274
277
|
max_tokens: 8192,
|
|
275
278
|
models: ["claude-opus-4-1-20250805", "claude-opus-4-20250514", "claude-sonnet-4-20250514", "claude-3-7-sonnet-20250219", "claude-3-haiku-20240307"],
|
|
276
279
|
keyName: "ANTHROPIC_API_KEY",
|
|
280
|
+
apiKeyNameOverride: "ANTHROPIC_API_TOKEN",
|
|
281
|
+
// GraphAI is currently using ANTHROPIC_API_KEY, but the official name is ANTHROPIC_API_TOKEN.
|
|
277
282
|
},
|
|
278
283
|
gemini: {
|
|
279
284
|
agentName: "geminiAgent",
|
package/lib/utils/utils.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* (No Node.js built-ins like fs, path, dotenv, etc.)
|
|
4
4
|
* Works in both Node.js and modern browsers.
|
|
5
5
|
*/
|
|
6
|
-
import { provider2LLMAgent } from "./provider2agent.js";
|
|
6
|
+
import { provider2LLMAgent, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, provider2SoundEffectAgent, provider2LipSyncAgent, } from "./provider2agent.js";
|
|
7
7
|
export const llmPair = (_llm, _model) => {
|
|
8
8
|
const llmKey = _llm ?? "openai";
|
|
9
9
|
const agent = provider2LLMAgent[llmKey]?.agentName ?? provider2LLMAgent.openai.agentName;
|
|
@@ -42,61 +42,30 @@ export const settings2GraphAIConfig = (settings, env) => {
|
|
|
42
42
|
const getKey = (prefix, key) => {
|
|
43
43
|
return settings?.[`${prefix}_${key}`] ?? settings?.[key] ?? env?.[`${prefix}_${key}`] ?? env?.[key];
|
|
44
44
|
};
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
imageReplicateAgent: {
|
|
58
|
-
apiKey: getKey("IMAGE", "REPLICATE_API_TOKEN"),
|
|
59
|
-
},
|
|
60
|
-
imageGenAIAgent: {
|
|
61
|
-
apiKey: getKey("IMAGE", "GEMINI_API_KEY"),
|
|
62
|
-
},
|
|
63
|
-
movieReplicateAgent: {
|
|
64
|
-
apiKey: getKey("MOVIE", "REPLICATE_API_TOKEN"),
|
|
65
|
-
},
|
|
66
|
-
movieGenAIAgent: {
|
|
67
|
-
apiKey: getKey("MOVIE", "GEMINI_API_KEY"),
|
|
68
|
-
},
|
|
69
|
-
ttsOpenaiAgent: {
|
|
70
|
-
apiKey: getKey("TTS", "OPENAI_API_KEY"),
|
|
71
|
-
baseURL: getKey("TTS", "OPENAI_BASE_URL"),
|
|
72
|
-
},
|
|
73
|
-
ttsNijivoiceAgent: {
|
|
74
|
-
apiKey: getKey("TTS", "NIJIVOICE_API_KEY"),
|
|
75
|
-
},
|
|
76
|
-
ttsGoogleAgent: {
|
|
77
|
-
apiKey: getKey("TTS", "GEMINI_API_KEY"),
|
|
78
|
-
},
|
|
79
|
-
ttsGeminiAgent: {
|
|
80
|
-
apiKey: getKey("TTS", "GEMINI_API_KEY"),
|
|
81
|
-
},
|
|
82
|
-
ttsElevenlabsAgent: {
|
|
83
|
-
apiKey: getKey("TTS", "ELEVENLABS_API_KEY"),
|
|
84
|
-
},
|
|
85
|
-
ttsKotodamaAgent: {
|
|
86
|
-
apiKey: getKey("TTS", "KOTODAMA_API_KEY"),
|
|
87
|
-
},
|
|
88
|
-
soundEffectReplicateAgent: {
|
|
89
|
-
apiKey: getKey("SOUND_EFFECT", "REPLICATE_API_TOKEN"),
|
|
90
|
-
},
|
|
91
|
-
lipSyncReplicateAgent: {
|
|
92
|
-
apiKey: getKey("LIPSYNC", "REPLICATE_API_TOKEN"),
|
|
93
|
-
},
|
|
94
|
-
// TODO
|
|
95
|
-
// browserlessAgent
|
|
96
|
-
// ttsGoogleAgent
|
|
97
|
-
// geminiAgent, groqAgent for tool
|
|
98
|
-
// TAVILY_API_KEY ( for deep research)
|
|
45
|
+
const addProviderConfigs = (config, providers, prefix) => {
|
|
46
|
+
Object.entries(providers).forEach(([__provider, info]) => {
|
|
47
|
+
if (info.agentName === "mediaMockAgent" || !info.keyName)
|
|
48
|
+
return;
|
|
49
|
+
const apiKeyName = info.apiKeyNameOverride || info.keyName;
|
|
50
|
+
config[info.agentName] = {
|
|
51
|
+
apiKey: getKey(prefix, apiKeyName),
|
|
52
|
+
};
|
|
53
|
+
if (info.baseURLKeyName) {
|
|
54
|
+
config[info.agentName].baseURL = getKey(prefix, info.baseURLKeyName);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
99
57
|
};
|
|
58
|
+
const config = {};
|
|
59
|
+
addProviderConfigs(config, provider2LLMAgent, "LLM");
|
|
60
|
+
addProviderConfigs(config, provider2TTSAgent, "TTS");
|
|
61
|
+
addProviderConfigs(config, provider2ImageAgent, "IMAGE");
|
|
62
|
+
addProviderConfigs(config, provider2MovieAgent, "MOVIE");
|
|
63
|
+
addProviderConfigs(config, provider2SoundEffectAgent, "SOUND_EFFECT");
|
|
64
|
+
addProviderConfigs(config, provider2LipSyncAgent, "LIPSYNC");
|
|
65
|
+
// TODO
|
|
66
|
+
// browserlessAgent
|
|
67
|
+
// geminiAgent, groqAgent for tool
|
|
68
|
+
// TAVILY_API_KEY ( for deep research)
|
|
100
69
|
return deepClean(config) ?? {};
|
|
101
70
|
};
|
|
102
71
|
export const deepClean = (input) => {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { MulmoVideoFilter } from "../types/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Convert video filter objects to FFmpeg filter strings
|
|
4
|
+
* @param filter - Video filter configuration object
|
|
5
|
+
* @returns FFmpeg filter string
|
|
6
|
+
*/
|
|
7
|
+
export declare const convertVideoFilterToFFmpeg: (filter: MulmoVideoFilter) => string;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// Helper functions for different filter categories
|
|
2
|
+
const convertColorFilter = (filter) => {
|
|
3
|
+
switch (filter.type) {
|
|
4
|
+
case "mono":
|
|
5
|
+
return "hue=s=0";
|
|
6
|
+
case "sepia":
|
|
7
|
+
return "colorchannelmixer=.393:.769:.189:0:.349:.686:.168:0:.272:.534:.131";
|
|
8
|
+
case "brightness_contrast":
|
|
9
|
+
return `eq=brightness=${filter.brightness}:contrast=${filter.contrast}`;
|
|
10
|
+
case "hue": {
|
|
11
|
+
const parts = [`h=${filter.hue ?? 0}`, `s=${filter.saturation ?? 1}`, `b=${filter.brightness ?? 0}`];
|
|
12
|
+
return `hue=${parts.join(":")}`;
|
|
13
|
+
}
|
|
14
|
+
case "colorbalance":
|
|
15
|
+
return `colorbalance=rs=${filter.rs}:gs=${filter.gs}:bs=${filter.bs}:rm=${filter.rm}:gm=${filter.gm}:bm=${filter.bm}:rh=${filter.rh}:gh=${filter.gh}:bh=${filter.bh}`;
|
|
16
|
+
case "vibrance":
|
|
17
|
+
return `vibrance=intensity=${filter.intensity}`;
|
|
18
|
+
case "negate":
|
|
19
|
+
return `negate${filter.negate_alpha ? "=negate_alpha=1" : ""}`;
|
|
20
|
+
case "colorhold":
|
|
21
|
+
return `colorhold=color=${filter.color}:similarity=${filter.similarity}:blend=${filter.blend}`;
|
|
22
|
+
case "colorkey":
|
|
23
|
+
return `colorkey=color=${filter.color}:similarity=${filter.similarity}:blend=${filter.blend}`;
|
|
24
|
+
default:
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
const convertBlurSharpenFilter = (filter) => {
|
|
29
|
+
switch (filter.type) {
|
|
30
|
+
case "blur":
|
|
31
|
+
return `boxblur=${filter.radius}:${filter.power}`;
|
|
32
|
+
case "gblur":
|
|
33
|
+
return `gblur=sigma=${filter.sigma}`;
|
|
34
|
+
case "avgblur":
|
|
35
|
+
return `avgblur=sizeX=${filter.sizeX}:sizeY=${filter.sizeY}`;
|
|
36
|
+
case "unsharp": {
|
|
37
|
+
const parts = [
|
|
38
|
+
`luma_msize_x=${filter.luma_msize_x ?? 5}`,
|
|
39
|
+
`luma_msize_y=${filter.luma_msize_y ?? 5}`,
|
|
40
|
+
`luma_amount=${filter.luma_amount ?? 1}`,
|
|
41
|
+
`chroma_msize_x=${filter.chroma_msize_x ?? 5}`,
|
|
42
|
+
`chroma_msize_y=${filter.chroma_msize_y ?? 5}`,
|
|
43
|
+
`chroma_amount=${filter.chroma_amount ?? 0}`,
|
|
44
|
+
];
|
|
45
|
+
return `unsharp=${parts.join(":")}`;
|
|
46
|
+
}
|
|
47
|
+
default:
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
const convertEdgeNoiseFilter = (filter) => {
|
|
52
|
+
switch (filter.type) {
|
|
53
|
+
case "edgedetect": {
|
|
54
|
+
const parts = [`low=${filter.low ?? 0.2}`, `high=${filter.high ?? 0.4}`, `mode=${filter.mode ?? "wires"}`];
|
|
55
|
+
return `edgedetect=${parts.join(":")}`;
|
|
56
|
+
}
|
|
57
|
+
case "sobel": {
|
|
58
|
+
const parts = [`planes=${filter.planes ?? 15}`, `scale=${filter.scale ?? 1}`, `delta=${filter.delta ?? 0}`];
|
|
59
|
+
return `sobel=${parts.join(":")}`;
|
|
60
|
+
}
|
|
61
|
+
case "emboss":
|
|
62
|
+
return "convolution='0 -1 0 -1 5 -1 0 -1 0:0 -1 0 -1 5 -1 0 -1 0:0 -1 0 -1 5 -1 0 -1 0:0 -1 0 -1 5 -1 0 -1 0'";
|
|
63
|
+
case "glitch":
|
|
64
|
+
if (filter.style === "blend") {
|
|
65
|
+
return `tblend=all_mode=difference,noise=alls=${filter.intensity}`;
|
|
66
|
+
}
|
|
67
|
+
return `noise=alls=${filter.intensity}:allf=t+u`;
|
|
68
|
+
case "grain":
|
|
69
|
+
return `noise=alls=${filter.intensity}:allf=t`;
|
|
70
|
+
default:
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
const convertTransformEffectFilter = (filter) => {
|
|
75
|
+
switch (filter.type) {
|
|
76
|
+
case "hflip":
|
|
77
|
+
return "hflip";
|
|
78
|
+
case "vflip":
|
|
79
|
+
return "vflip";
|
|
80
|
+
case "rotate":
|
|
81
|
+
return `rotate=angle=${filter.angle}:fillcolor=${filter.fillcolor}`;
|
|
82
|
+
case "transpose": {
|
|
83
|
+
const dirMap = { cclock: "0", clock: "1", cclock_flip: "2", clock_flip: "3" };
|
|
84
|
+
return `transpose=dir=${dirMap[filter.dir]}`;
|
|
85
|
+
}
|
|
86
|
+
case "vignette": {
|
|
87
|
+
const parts = [`angle=${filter.angle ?? Math.PI / 5}`];
|
|
88
|
+
if (filter.x0 !== undefined)
|
|
89
|
+
parts.push(`x0=${filter.x0}`);
|
|
90
|
+
if (filter.y0 !== undefined)
|
|
91
|
+
parts.push(`y0=${filter.y0}`);
|
|
92
|
+
parts.push(`mode=${filter.mode ?? "forward"}`);
|
|
93
|
+
return `vignette=${parts.join(":")}`;
|
|
94
|
+
}
|
|
95
|
+
case "fade":
|
|
96
|
+
return `fade=type=${filter.mode}:start_frame=${filter.start_frame}:nb_frames=${filter.nb_frames}${filter.alpha ? ":alpha=1" : ""}:color=${filter.color}`;
|
|
97
|
+
case "pixelize":
|
|
98
|
+
return `scale=iw/${filter.width}:ih/${filter.height},scale=${filter.width}*iw:${filter.height}*ih:flags=neighbor`;
|
|
99
|
+
case "pseudocolor":
|
|
100
|
+
return `pseudocolor=preset=${filter.preset}`;
|
|
101
|
+
default:
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const convertTemporalDistortionFilter = (filter) => {
|
|
106
|
+
switch (filter.type) {
|
|
107
|
+
case "tmix": {
|
|
108
|
+
const weights = filter.weights ? `:weights=${filter.weights}` : "";
|
|
109
|
+
return `tmix=frames=${filter.frames}${weights}`;
|
|
110
|
+
}
|
|
111
|
+
case "lagfun": {
|
|
112
|
+
const parts = [`decay=${filter.decay ?? 0.95}`, `planes=${filter.planes ?? 15}`];
|
|
113
|
+
return `lagfun=${parts.join(":")}`;
|
|
114
|
+
}
|
|
115
|
+
case "threshold":
|
|
116
|
+
return `threshold=planes=${filter.planes}`;
|
|
117
|
+
case "elbg":
|
|
118
|
+
return `elbg=l=${filter.codebook_length}`;
|
|
119
|
+
case "lensdistortion":
|
|
120
|
+
return `lenscorrection=k1=${filter.k1}:k2=${filter.k2}`;
|
|
121
|
+
case "chromashift": {
|
|
122
|
+
const parts = [`cbh=${filter.cbh ?? 0}`, `cbv=${filter.cbv ?? 0}`, `crh=${filter.crh ?? 0}`, `crv=${filter.crv ?? 0}`, `edge=${filter.edge ?? "smear"}`];
|
|
123
|
+
return `chromashift=${parts.join(":")}`;
|
|
124
|
+
}
|
|
125
|
+
case "deflicker":
|
|
126
|
+
return `deflicker=size=${filter.size}:mode=${filter.mode}`;
|
|
127
|
+
case "dctdnoiz":
|
|
128
|
+
return `dctdnoiz=sigma=${filter.sigma}`;
|
|
129
|
+
default:
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
/**
|
|
134
|
+
* Convert video filter objects to FFmpeg filter strings
|
|
135
|
+
* @param filter - Video filter configuration object
|
|
136
|
+
* @returns FFmpeg filter string
|
|
137
|
+
*/
|
|
138
|
+
export const convertVideoFilterToFFmpeg = (filter) => {
|
|
139
|
+
if (filter.type === "custom") {
|
|
140
|
+
return filter.filter;
|
|
141
|
+
}
|
|
142
|
+
// Try each category converter
|
|
143
|
+
return (convertColorFilter(filter) ||
|
|
144
|
+
convertBlurSharpenFilter(filter) ||
|
|
145
|
+
convertEdgeNoiseFilter(filter) ||
|
|
146
|
+
convertTransformEffectFilter(filter) ||
|
|
147
|
+
convertTemporalDistortionFilter(filter) ||
|
|
148
|
+
"");
|
|
149
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "2.0
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.node.js",
|
|
@@ -96,11 +96,11 @@
|
|
|
96
96
|
"graphai": "^2.0.16",
|
|
97
97
|
"jsdom": "^27.2.0",
|
|
98
98
|
"marked": "^17.0.1",
|
|
99
|
-
"mulmocast-vision": "^1.0.
|
|
99
|
+
"mulmocast-vision": "^1.0.8",
|
|
100
100
|
"ora": "^9.0.0",
|
|
101
|
-
"puppeteer": "^24.
|
|
101
|
+
"puppeteer": "^24.32.0",
|
|
102
102
|
"replicate": "^1.4.0",
|
|
103
|
-
"yaml": "^2.8.
|
|
103
|
+
"yaml": "^2.8.2",
|
|
104
104
|
"yargs": "^18.0.0",
|
|
105
105
|
"zod": "^4.1.13"
|
|
106
106
|
},
|
|
@@ -114,10 +114,10 @@
|
|
|
114
114
|
"eslint-config-prettier": "^10.1.8",
|
|
115
115
|
"eslint-plugin-prettier": "^5.5.4",
|
|
116
116
|
"eslint-plugin-sonarjs": "^3.0.5",
|
|
117
|
-
"prettier": "^3.7.
|
|
118
|
-
"tsx": "^4.
|
|
117
|
+
"prettier": "^3.7.4",
|
|
118
|
+
"tsx": "^4.21.0",
|
|
119
119
|
"typescript": "^5.9.3",
|
|
120
|
-
"typescript-eslint": "^8.48.
|
|
120
|
+
"typescript-eslint": "^8.48.1"
|
|
121
121
|
},
|
|
122
122
|
"engines": {
|
|
123
123
|
"node": ">=20.0.0"
|
package/scripts/test/README.md
CHANGED
|
@@ -12,10 +12,10 @@ This directory contains MulmoScript samples for testing MulmoCast features.
|
|
|
12
12
|
|
|
13
13
|
Simple test scripts for basic functionality verification
|
|
14
14
|
|
|
15
|
-
- **test_hello.json** - 最もシンプルなHello Worldテスト / Simplest Hello World test
|
|
16
|
-
- **test.json** - 基本的な動作テスト / Basic functionality test
|
|
17
|
-
- **test1.json
|
|
18
|
-
- **test_beats.json** - Beatの基本機能テスト / Beat basic features test
|
|
15
|
+
- [**test_hello.json**](./test_hello.json) - 最もシンプルなHello Worldテスト / Simplest Hello World test
|
|
16
|
+
- [**test.json**](./test.json) - 基本的な動作テスト / Basic functionality test
|
|
17
|
+
- [**test1.json**](./test1.json), [**test2.json**](./test2.json) - 追加の基本テスト / Additional basic tests
|
|
18
|
+
- [**test_beats.json**](./test_beats.json) - Beatの基本機能テスト / Beat basic features test
|
|
19
19
|
|
|
20
20
|
### 🎤 TTS(音声合成)テスト / TTS (Text-to-Speech) Tests
|
|
21
21
|
|
|
@@ -23,13 +23,13 @@ Simple test scripts for basic functionality verification
|
|
|
23
23
|
|
|
24
24
|
Tests for various TTS providers
|
|
25
25
|
|
|
26
|
-
- **test_all_tts.json** - 全TTSプロバイダーのテスト(OpenAI, Gemini, Google, ElevenLabs, Nijivoice) / All TTS providers test
|
|
27
|
-
- **test_audio.json** - 音声パラメータのテスト(padding, duration, movieVolumeなど) / Audio parameters test
|
|
28
|
-
- **test_audio_gemini.json** - Gemini TTSの個別テスト / Gemini TTS specific test
|
|
29
|
-
- **test_audio_instructions.json** - OpenAI TTS instructionsのテスト / OpenAI TTS instructions test
|
|
30
|
-
- **test_elevenlabs_models.json** - ElevenLabsの複数モデルテスト / ElevenLabs multiple models test
|
|
31
|
-
- **test_voices.json** - 複数の音声設定テスト / Multiple voice settings test
|
|
32
|
-
- **test_mixed_providers.json** - 複数のTTSプロバイダー混在テスト / Mixed TTS providers test
|
|
26
|
+
- [**test_all_tts.json**](./test_all_tts.json) - 全TTSプロバイダーのテスト(OpenAI, Gemini, Google, ElevenLabs, Nijivoice) / All TTS providers test
|
|
27
|
+
- [**test_audio.json**](./test_audio.json) - 音声パラメータのテスト(padding, duration, movieVolumeなど) / Audio parameters test
|
|
28
|
+
- [**test_audio_gemini.json**](./test_audio_gemini.json) - Gemini TTSの個別テスト / Gemini TTS specific test
|
|
29
|
+
- [**test_audio_instructions.json**](./test_audio_instructions.json) - OpenAI TTS instructionsのテスト / OpenAI TTS instructions test
|
|
30
|
+
- [**test_elevenlabs_models.json**](./test_elevenlabs_models.json) - ElevenLabsの複数モデルテスト / ElevenLabs multiple models test
|
|
31
|
+
- [**test_voices.json**](./test_voices.json) - 複数の音声設定テスト / Multiple voice settings test
|
|
32
|
+
- [**test_mixed_providers.json**](./test_mixed_providers.json) - 複数のTTSプロバイダー混在テスト / Mixed TTS providers test
|
|
33
33
|
|
|
34
34
|
### 🖼️ 画像生成テスト / Image Generation Tests
|
|
35
35
|
|
|
@@ -37,13 +37,13 @@ Tests for various TTS providers
|
|
|
37
37
|
|
|
38
38
|
Image generation feature tests
|
|
39
39
|
|
|
40
|
-
- **test_images.json** - 画像生成の基本テスト / Basic image generation test
|
|
41
|
-
- **test_hello_image.json** - Hello World画像テスト / Hello World image test
|
|
42
|
-
- **test_image_refs.json** - 参照画像を使った生成テスト / Image generation with references
|
|
43
|
-
- **test_markdown.json** - Markdown形式の画像テスト / Markdown format image test
|
|
44
|
-
- **test_html.json** - HTMLから画像生成テスト / HTML to image test
|
|
45
|
-
- **test_vision.json** - Vision APIを使った画像生成テスト / Vision API image test
|
|
46
|
-
- **test_layout.json** - レイアウト機能のテスト / Layout features test
|
|
40
|
+
- [**test_images.json**](./test_images.json) - 画像生成の基本テスト / Basic image generation test
|
|
41
|
+
- [**test_hello_image.json**](./test_hello_image.json) - Hello World画像テスト / Hello World image test
|
|
42
|
+
- [**test_image_refs.json**](./test_image_refs.json) - 参照画像を使った生成テスト / Image generation with references
|
|
43
|
+
- [**test_markdown.json**](./test_markdown.json) - Markdown形式の画像テスト / Markdown format image test
|
|
44
|
+
- [**test_html.json**](./test_html.json) - HTMLから画像生成テスト / HTML to image test
|
|
45
|
+
- [**test_vision.json**](./test_vision.json) - Vision APIを使った画像生成テスト / Vision API image test
|
|
46
|
+
- [**test_layout.json**](./test_layout.json) - レイアウト機能のテスト / Layout features test
|
|
47
47
|
|
|
48
48
|
### 🎬 動画生成テスト / Video Generation Tests
|
|
49
49
|
|
|
@@ -51,12 +51,12 @@ Image generation feature tests
|
|
|
51
51
|
|
|
52
52
|
Video generation feature tests
|
|
53
53
|
|
|
54
|
-
- **test_movie.json** - 動画生成の基本テスト(imagePrompt + moviePrompt) / Basic video generation test
|
|
55
|
-
- **test_movie2.json** - 動画生成の追加テスト / Additional video generation test
|
|
56
|
-
- **test_genai_movie.json** - GenAI動画生成テスト / GenAI video generation test
|
|
57
|
-
- **test_genai.json** - GenAI機能テスト / GenAI features test
|
|
58
|
-
- **test_replicate.json** - Replicate動画生成テスト / Replicate video generation test
|
|
59
|
-
- **test_mv.json** - ミュージックビデオ形式のテスト / Music video format test
|
|
54
|
+
- [**test_movie.json**](./test_movie.json) - 動画生成の基本テスト(imagePrompt + moviePrompt) / Basic video generation test
|
|
55
|
+
- [**test_movie2.json**](./test_movie2.json) - 動画生成の追加テスト / Additional video generation test
|
|
56
|
+
- [**test_genai_movie.json**](./test_genai_movie.json) - GenAI動画生成テスト / GenAI video generation test
|
|
57
|
+
- [**test_genai.json**](./test_genai.json) - GenAI機能テスト / GenAI features test
|
|
58
|
+
- [**test_replicate.json**](./test_replicate.json) - Replicate動画生成テスト / Replicate video generation test
|
|
59
|
+
- [**test_mv.json**](./test_mv.json) - ミュージックビデオ形式のテスト / Music video format test
|
|
60
60
|
|
|
61
61
|
### 🎭 高度な機能テスト / Advanced Feature Tests
|
|
62
62
|
|
|
@@ -64,17 +64,17 @@ Video generation feature tests
|
|
|
64
64
|
|
|
65
65
|
Special features and complex scenario tests
|
|
66
66
|
|
|
67
|
-
- **test_spillover.json** - 音声スピルオーバー機能テスト / Audio spillover feature test
|
|
68
|
-
- **test_lipsync.json** - リップシンク機能テスト / Lip-sync feature test
|
|
69
|
-
- **test_transition.json** - トランジション効果テスト / Transition effects test
|
|
70
|
-
- **test_transition_no_audio.json** - 音声なしトランジションテスト / Transition without audio test
|
|
71
|
-
- **test_slideout_left_no_audio.json** - スライドアウト効果テスト / Slide-out effect test
|
|
72
|
-
- **test_sound_effect.json** - サウンドエフェクトテスト / Sound effect test
|
|
73
|
-
- **test_voice_over.json** - ボイスオーバー機能テスト / Voice-over feature test
|
|
74
|
-
- **test_captions.json** - 字幕機能テスト / Caption feature test
|
|
75
|
-
- **test_hello_caption.json** - Hello World字幕テスト / Hello World caption test
|
|
76
|
-
- **test_loop.json** - ループ再生テスト / Loop playback test
|
|
77
|
-
- **test_video_speed.json** - 動画速度調整テスト / Video speed adjustment test
|
|
67
|
+
- [**test_spillover.json**](./test_spillover.json) - 音声スピルオーバー機能テスト / Audio spillover feature test
|
|
68
|
+
- [**test_lipsync.json**](./test_lipsync.json) - リップシンク機能テスト / Lip-sync feature test
|
|
69
|
+
- [**test_transition.json**](./test_transition.json) - トランジション効果テスト / Transition effects test
|
|
70
|
+
- [**test_transition_no_audio.json**](./test_transition_no_audio.json) - 音声なしトランジションテスト / Transition without audio test
|
|
71
|
+
- [**test_slideout_left_no_audio.json**](./test_slideout_left_no_audio.json) - スライドアウト効果テスト / Slide-out effect test
|
|
72
|
+
- [**test_sound_effect.json**](./test_sound_effect.json) - サウンドエフェクトテスト / Sound effect test
|
|
73
|
+
- [**test_voice_over.json**](./test_voice_over.json) - ボイスオーバー機能テスト / Voice-over feature test
|
|
74
|
+
- [**test_captions.json**](./test_captions.json) - 字幕機能テスト / Caption feature test
|
|
75
|
+
- [**test_hello_caption.json**](./test_hello_caption.json) - Hello World字幕テスト / Hello World caption test
|
|
76
|
+
- [**test_loop.json**](./test_loop.json) - ループ再生テスト / Loop playback test
|
|
77
|
+
- [**test_video_speed.json**](./test_video_speed.json) - 動画速度調整テスト / Video speed adjustment test
|
|
78
78
|
|
|
79
79
|
### 🔧 特殊条件テスト / Special Condition Tests
|
|
80
80
|
|
|
@@ -82,13 +82,13 @@ Special features and complex scenario tests
|
|
|
82
82
|
|
|
83
83
|
Edge cases and special condition tests
|
|
84
84
|
|
|
85
|
-
- **test_no_audio.json** - 音声なし動画テスト / Video without audio test
|
|
86
|
-
- **test_no_audio_with_credit.json** - クレジット付き音声なしテスト / No audio with credits test
|
|
87
|
-
- **test_hello_nobgm.json** - BGMなしテスト / Test without BGM
|
|
88
|
-
- **test_size_error.json** - サイズエラーテスト / Size error test
|
|
89
|
-
- **test_media.json** - メディアファイル処理テスト / Media file processing test
|
|
90
|
-
- **test_order.json** - 順序処理テスト / Order processing test
|
|
91
|
-
- **test_order_portrait.json** - 縦向き順序テスト / Portrait order test
|
|
85
|
+
- [**test_no_audio.json**](./test_no_audio.json) - 音声なし動画テスト / Video without audio test
|
|
86
|
+
- [**test_no_audio_with_credit.json**](./test_no_audio_with_credit.json) - クレジット付き音声なしテスト / No audio with credits test
|
|
87
|
+
- [**test_hello_nobgm.json**](./test_hello_nobgm.json) - BGMなしテスト / Test without BGM
|
|
88
|
+
- [**test_size_error.json**](./test_size_error.json) - サイズエラーテスト / Size error test
|
|
89
|
+
- [**test_media.json**](./test_media.json) - メディアファイル処理テスト / Media file processing test
|
|
90
|
+
- [**test_order.json**](./test_order.json) - 順序処理テスト / Order processing test
|
|
91
|
+
- [**test_order_portrait.json**](./test_order_portrait.json) - 縦向き順序テスト / Portrait order test
|
|
92
92
|
|
|
93
93
|
### 🌍 多言語テスト / Multi-language Tests
|
|
94
94
|
|
|
@@ -96,8 +96,8 @@ Edge cases and special condition tests
|
|
|
96
96
|
|
|
97
97
|
Language setting tests
|
|
98
98
|
|
|
99
|
-
- **test_lang.json** - 多言語サポートテスト / Multi-language support test
|
|
100
|
-
- **test_en.json** - 英語専用テスト / English-only test
|
|
99
|
+
- [**test_lang.json**](./test_lang.json) - 多言語サポートテスト / Multi-language support test
|
|
100
|
+
- [**test_en.json**](./test_en.json) - 英語専用テスト / English-only test
|
|
101
101
|
|
|
102
102
|
### 🎯 プロバイダー別テスト / Provider-Specific Tests
|
|
103
103
|
|
|
@@ -105,10 +105,10 @@ Language setting tests
|
|
|
105
105
|
|
|
106
106
|
Provider-specific feature tests
|
|
107
107
|
|
|
108
|
-
- **test_hello_google.json** - Google TTS専用テスト / Google TTS specific test
|
|
109
|
-
- **gpt.json** - GPTモデルテスト / GPT model test
|
|
110
|
-
- **mulmo_story.json** - ストーリー形式テスト / Story format test
|
|
111
|
-
- **nano_banana.json** - カスタムサンプル / Custom sample
|
|
108
|
+
- [**test_hello_google.json**](./test_hello_google.json) - Google TTS専用テスト / Google TTS specific test
|
|
109
|
+
- [**gpt.json**](./gpt.json) - GPTモデルテスト / GPT model test
|
|
110
|
+
- [**mulmo_story.json**](./mulmo_story.json) - ストーリー形式テスト / Story format test
|
|
111
|
+
- [**nano_banana.json**](./nano_banana.json) - カスタムサンプル / Custom sample
|
|
112
112
|
|
|
113
113
|
## 🚀 使い方 / Usage
|
|
114
114
|
|