mulmocast 2.17.0 → 2.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +110 -38
  2. package/assets/schemas/mulmo_script.json +100 -0
  3. package/lib/actions/audio.js +4 -1
  4. package/lib/actions/avatar.js +4 -1
  5. package/lib/actions/captions.js +1 -1
  6. package/lib/actions/image_agents.d.ts +1 -0
  7. package/lib/actions/image_agents.js +2 -2
  8. package/lib/actions/images.d.ts +2 -0
  9. package/lib/actions/images.js +1 -0
  10. package/lib/actions/pdf.js +23 -2
  11. package/lib/agents/add_bgm_agent.d.ts +13 -1
  12. package/lib/agents/add_bgm_agent.js +77 -4
  13. package/lib/agents/movie_genai_agent.d.ts +24 -0
  14. package/lib/agents/movie_genai_agent.js +56 -43
  15. package/lib/cli/bin.js +0 -0
  16. package/lib/cli/commands/bundle/builder.d.ts +2 -0
  17. package/lib/cli/commands/html/builder.d.ts +2 -0
  18. package/lib/cli/commands/markdown/builder.d.ts +2 -0
  19. package/lib/cli/commands/viewer/builder.d.ts +2 -0
  20. package/lib/cli/common.d.ts +2 -0
  21. package/lib/cli/common.js +5 -0
  22. package/lib/cli/helpers.js +3 -1
  23. package/lib/mcp/server.js +0 -0
  24. package/lib/methods/mulmo_studio_context.d.ts +6 -0
  25. package/lib/methods/mulmo_studio_context.js +26 -0
  26. package/lib/types/provider2agent.d.ts +1 -4
  27. package/lib/types/provider2agent.js +27 -24
  28. package/lib/types/schema.d.ts +64 -0
  29. package/lib/types/schema.js +42 -0
  30. package/lib/types/type.d.ts +4 -1
  31. package/lib/utils/context.d.ts +32 -0
  32. package/lib/utils/estimate_usage.js +6 -5
  33. package/lib/utils/gemini_omni_video.d.ts +49 -0
  34. package/lib/utils/gemini_omni_video.js +36 -0
  35. package/lib/utils/html_render.d.ts +6 -5
  36. package/lib/utils/html_render.js +37 -30
  37. package/lib/utils/image_plugins/chart.js +2 -1
  38. package/lib/utils/image_plugins/html_tailwind.js +5 -4
  39. package/lib/utils/image_plugins/markdown.js +2 -1
  40. package/lib/utils/image_plugins/mermaid.js +2 -1
  41. package/lib/utils/image_plugins/slide.js +2 -1
  42. package/lib/utils/image_plugins/text_slide.js +2 -1
  43. package/lib/utils/image_plugins/vision.js +7 -0
  44. package/lib/utils/jinglescript.d.ts +12 -0
  45. package/lib/utils/jinglescript.js +63 -0
  46. package/lib/utils/render_network_guard.d.ts +2 -0
  47. package/lib/utils/render_network_guard.js +36 -0
  48. package/lib/utils/render_network_policy.d.ts +15 -0
  49. package/lib/utils/render_network_policy.js +85 -0
  50. package/package.json +5 -4
  51. package/scripts/test/test_genai.json +10 -9
  52. package/scripts/test/test_genai_movie.json +1 -1
  53. package/scripts/test/test_image_references.json +3 -2
  54. package/scripts/test/test_jinglescript_sound_effects.json +101 -0
  55. package/scripts/test/test_movie_references.json +2 -1
  56. package/scripts/test/test_sound_effects.json +64 -0
  57. package/scripts/test/test_sound_effects_animation.json +441 -0
  58. package/scripts/test/test_veo31_lite.json +3 -3
  59. package/scripts/test/test_captions_2.json~ +0 -278
  60. package/scripts/test/test_en_portrait.json~ +0 -31
  61. package/scripts/test/test_slide_image_ref_gemini_en.json~ +0 -289
  62. package/scripts/test/test_vertexai.json~ +0 -21
@@ -2,6 +2,7 @@ import { renderMarkdownToImage } from "../html_render.js";
2
2
  import { parrotingImagePath } from "./utils.js";
3
3
  import { resolveCombinedStyle } from "./bg_image_util.js";
4
4
  import { marked } from "marked";
5
+ import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
5
6
  export const imageType = "textSlide";
6
7
  const processTextSlide = async (params) => {
7
8
  const { beat, imagePath, canvasSize } = params;
@@ -17,7 +18,7 @@ const processTextSlide = async (params) => {
17
18
  const marginTop = slide.subtitle ? canvasSize.height * 0.4 : canvasSize.height * 0.45;
18
19
  return `body {margin-top: ${marginTop}px;}`;
19
20
  })();
20
- await renderMarkdownToImage(markdown, combinedStyle + topMargin, imagePath, canvasSize.width, canvasSize.height);
21
+ await renderMarkdownToImage(markdown, combinedStyle + topMargin, imagePath, canvasSize.width, canvasSize.height, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
21
22
  return imagePath;
22
23
  };
23
24
  const dumpMarkdown = (params) => {
@@ -1,12 +1,19 @@
1
1
  import { parrotingImagePath } from "./utils.js";
2
2
  import { htmlPlugin, templateNameTofunctionName } from "mulmocast-vision";
3
3
  import { resolve as resolvePath } from "path";
4
+ import { agentGenerationError, imageAction, unsupportedImageTypeTarget } from "../error_cause.js";
4
5
  export const imageType = "vision";
5
6
  const processVision = async (params) => {
6
7
  const { beat, imagePath, context } = params;
7
8
  const rootDir = context.fileDirs.nodeModuleRootPath ? resolvePath(context.fileDirs.nodeModuleRootPath, "mulmocast-vision") : undefined;
8
9
  if (!beat?.image || beat.image.type !== imageType)
9
10
  return;
11
+ // mulmocast-vision launches its own browser, which the strict network guard cannot reach.
12
+ if (context.strictNetwork) {
13
+ throw new Error("vision beats cannot be rendered with --strict-network: mulmocast-vision renders in its own browser, outside the network guard", {
14
+ cause: agentGenerationError("visionPlugin", imageAction, unsupportedImageTypeTarget),
15
+ });
16
+ }
10
17
  const handler = new htmlPlugin({ rootDir });
11
18
  await handler.callNamedFunction(templateNameTofunctionName(beat.image.style), beat.image.data, {
12
19
  functionName: beat.image.style,
@@ -0,0 +1,12 @@
1
+ import type { MulmoStudioContext } from "../types/index.js";
2
+ type JingleScore = Record<string, unknown>;
3
+ export declare const getJingleFilePath: (score: JingleScore, context: MulmoStudioContext) => string;
4
+ export declare const getJingleScores: (context: MulmoStudioContext) => {
5
+ score: Record<string, unknown>;
6
+ beatIndex: number;
7
+ effectIndex: number;
8
+ }[];
9
+ export declare const validateJingleScores: (context: MulmoStudioContext) => void;
10
+ export declare const renderJingleScores: (context: MulmoStudioContext) => Promise<void>;
11
+ export declare const writeFileAtomically: (filePath: string, data: Uint8Array) => void;
12
+ export {};
@@ -0,0 +1,63 @@
1
+ import fs from "fs";
2
+ import path from "path";
3
+ import { GraphAILogger } from "graphai";
4
+ import { checkScore, formatProblem, loadSamples, parseScore, render, toWav } from "jinglescript";
5
+ import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
6
+ import { hashSHA256, mkdir } from "./file.js";
7
+ import { userAssert } from "./utils.js";
8
+ // The audio mix (addBGMAgent) runs at 44.1 kHz.
9
+ const JINGLE_SAMPLE_RATE = 44100;
10
+ // Per download of a recorded sample (grandpiano: about 1.2 MB per note).
11
+ const SAMPLE_FETCH_TIMEOUT_MS = 60000;
12
+ // Rendered jingles are cached by the hash of their score, so identical scores share one file.
13
+ export const getJingleFilePath = (score, context) => {
14
+ return path.join(MulmoStudioContextMethods.getAudioDirPath(context), `jingle_${hashSHA256(JSON.stringify(score))}.wav`);
15
+ };
16
+ export const getJingleScores = (context) => {
17
+ return context.studio.script.beats.flatMap((beat, beatIndex) => (beat.soundEffects ?? []).flatMap((soundEffect, effectIndex) => soundEffect.source.kind === "jinglescript" ? [{ score: soundEffect.source.score, beatIndex, effectIndex }] : []));
18
+ };
19
+ // Checks every JingleScript score in the script; throws listing every problem with its location.
20
+ export const validateJingleScores = (context) => {
21
+ const problems = getJingleScores(context).flatMap(({ score, beatIndex, effectIndex }) => checkScore(score).errors.map((problem) => `beats[${beatIndex}].soundEffects[${effectIndex}].source.score: ${formatProblem(problem).trim()}`));
22
+ userAssert(problems.length === 0, `Invalid JingleScript score:\n${problems.join("\n")}`);
23
+ };
24
+ const fetchWithTimeout = (input, init) => fetch(input, { ...init, signal: AbortSignal.timeout(SAMPLE_FETCH_TIMEOUT_MS) });
25
+ // Sampled instruments (grandpiano) play recordings that jinglescript downloads on first use and
26
+ // caches; a score without them loads nothing.
27
+ const loadJingleSamples = async (score, location) => {
28
+ try {
29
+ return await loadSamples(score, { fetch: fetchWithTimeout });
30
+ }
31
+ catch (error) {
32
+ throw new Error(`${location}: failed to load JingleScript samples: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
33
+ }
34
+ };
35
+ // Renders the JingleScript scores of the script to WAV files (skipping cached ones unless forced).
36
+ export const renderJingleScores = async (context) => {
37
+ validateJingleScores(context);
38
+ for (const { score, beatIndex, effectIndex } of getJingleScores(context)) {
39
+ const filePath = getJingleFilePath(score, context);
40
+ if (!context.force && fs.existsSync(filePath)) {
41
+ continue;
42
+ }
43
+ const parsed = parseScore(score);
44
+ const samples = await loadJingleSamples(parsed, `beats[${beatIndex}].soundEffects[${effectIndex}].source.score`);
45
+ const { audio, sampleRate } = render(parsed, { sampleRate: JINGLE_SAMPLE_RATE, samples });
46
+ mkdir(path.dirname(filePath));
47
+ writeFileAtomically(filePath, toWav(audio, sampleRate));
48
+ GraphAILogger.info(`jinglescript: rendered ${filePath}`);
49
+ }
50
+ };
51
+ // Writes to a temporary file next to the target, then renames it, so an interrupted write never
52
+ // leaves a partial WAV that a later run would take for a cached one.
53
+ export const writeFileAtomically = (filePath, data) => {
54
+ const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
55
+ try {
56
+ fs.writeFileSync(tmpPath, data);
57
+ fs.renameSync(tmpPath, filePath);
58
+ }
59
+ catch (error) {
60
+ fs.rmSync(tmpPath, { force: true });
61
+ throw error;
62
+ }
63
+ };
@@ -0,0 +1,2 @@
1
+ import type { Page } from "puppeteer";
2
+ export declare const guardRenderPage: (page: Page, allowedFileRoots: readonly string[]) => Promise<void>;
@@ -0,0 +1,36 @@
1
+ import fs from "node:fs";
2
+ import { GraphAILogger } from "graphai";
3
+ import { WEBRTC_REMOVAL_SCRIPT } from "./remotion/network_policy.js";
4
+ import { POPUP_BLOCK_SCRIPT, isAllowedRenderRequest, realRoot } from "./render_network_policy.js";
5
+ const fileRealPath = (filePath) => {
6
+ try {
7
+ return fs.realpathSync.native(filePath);
8
+ }
9
+ catch {
10
+ return undefined;
11
+ }
12
+ };
13
+ // Must run before the page loads anything. The WebRTC removal only reaches documents created after this call,
14
+ // which is why strict mode always navigates to a file instead of using setContent.
15
+ export const guardRenderPage = async (page, allowedFileRoots) => {
16
+ const realRoots = allowedFileRoots.map((root) => realRoot(root, fileRealPath));
17
+ const reported = new Set();
18
+ await page.setRequestInterception(true);
19
+ page.on("request", (request) => {
20
+ if (request.isInterceptResolutionHandled())
21
+ return;
22
+ const url = request.url();
23
+ // The page can close while a request is still pending; that rejection is not a failure of the render.
24
+ if (isAllowedRenderRequest(url, realRoots, fileRealPath)) {
25
+ request.continue().catch(() => undefined);
26
+ return;
27
+ }
28
+ if (!reported.has(url)) {
29
+ reported.add(url);
30
+ GraphAILogger.info(`strict network: blocked ${url}`);
31
+ }
32
+ request.abort("blockedbyclient").catch(() => undefined);
33
+ });
34
+ await page.evaluateOnNewDocument(WEBRTC_REMOVAL_SCRIPT);
35
+ await page.evaluateOnNewDocument(POPUP_BLOCK_SCRIPT);
36
+ };
@@ -0,0 +1,15 @@
1
+ export declare const RENDER_CDN_HOSTS: string[];
2
+ export declare const RENDER_JSDELIVR_PATHS: string[];
3
+ export type RenderNetworkOptions = {
4
+ strictNetwork?: boolean;
5
+ allowedFileRoots?: readonly string[];
6
+ };
7
+ export declare const STRICT_NETWORK_ENV = "MULMO_STRICT_NETWORK";
8
+ export declare const strictNetworkFromEnv: (env: Record<string, string | undefined>) => boolean;
9
+ export type RealPath = (filePath: string) => string | undefined;
10
+ export declare const isAllowedRenderRequest: (url: string, realRoots?: readonly string[], realPath?: RealPath) => boolean;
11
+ export declare const realRoot: (root: string, realPath: RealPath) => string;
12
+ export declare const strictNetworkLaunchArgs: () => string[];
13
+ export declare const POPUP_BLOCK_SCRIPT = "Object.defineProperty(globalThis, \"open\", { value: () => null, writable: false, configurable: false });";
14
+ export declare const RENDER_CONTENT_SECURITY_POLICY = "connect-src 'self' data: blob:";
15
+ export declare const withRenderContentSecurityPolicy: (html: string) => string;
@@ -0,0 +1,85 @@
1
+ import nodePath from "node:path";
2
+ import { fileURLToPath } from "node:url";
3
+ // What an HTML renderer's page may reach in strict network mode: files under the allowed roots, in-memory
4
+ // URLs, and the CDNs our own templates load from. Kept free of I/O so the rules can be tested without a browser.
5
+ export const RENDER_CDN_HOSTS = ["cdn.tailwindcss.com", "cdn.jsdelivr.net", "fonts.googleapis.com", "fonts.gstatic.com"];
6
+ // jsDelivr serves any file of any package and publishes per-version and per-file hit counts, so a page free to
7
+ // choose the path could signal through them; only the exact files our templates load are allowed.
8
+ export const RENDER_JSDELIVR_PATHS = [
9
+ "/npm/mermaid/dist/mermaid.min.js",
10
+ "/npm/mermaid@10/dist/mermaid.min.js",
11
+ "/npm/chart.js",
12
+ "/npm/chart.js@4",
13
+ "/npm/chartjs-chart-sankey",
14
+ "/npm/chartjs-chart-treemap@3",
15
+ ];
16
+ const JSDELIVR_HOST = "cdn.jsdelivr.net";
17
+ const IN_MEMORY_SCHEMES = ["data:", "blob:", "about:"];
18
+ export const STRICT_NETWORK_ENV = "MULMO_STRICT_NETWORK";
19
+ export const strictNetworkFromEnv = (env) => ["1", "true"].includes(env[STRICT_NETWORK_ENV] ?? "");
20
+ const parseUrl = (url) => {
21
+ try {
22
+ return new URL(url);
23
+ }
24
+ catch {
25
+ return undefined;
26
+ }
27
+ };
28
+ const filePathOf = (parsed) => {
29
+ try {
30
+ return fileURLToPath(parsed);
31
+ }
32
+ catch {
33
+ return undefined;
34
+ }
35
+ };
36
+ const isWithin = (root, filePath) => {
37
+ const relative = nodePath.relative(root, filePath);
38
+ return relative === "" || (!relative.startsWith("..") && !nodePath.isAbsolute(relative));
39
+ };
40
+ // The URL parser has already collapsed dot segments (encoded or not); the roots must already be real paths.
41
+ const isInsideFileRoots = (parsed, realRoots, realPath) => {
42
+ const filePath = filePathOf(parsed);
43
+ const realFilePath = filePath === undefined ? undefined : realPath(filePath);
44
+ return realFilePath !== undefined && realRoots.some((root) => isWithin(root, realFilePath));
45
+ };
46
+ const identityRealPath = (filePath) => nodePath.resolve(filePath);
47
+ const isAllowedCdnUrl = (parsed) => {
48
+ if (parsed.protocol !== "https:" || parsed.port !== "" || !RENDER_CDN_HOSTS.includes(parsed.hostname))
49
+ return false;
50
+ return parsed.hostname !== JSDELIVR_HOST || (parsed.search === "" && RENDER_JSDELIVR_PATHS.includes(parsed.pathname));
51
+ };
52
+ export const isAllowedRenderRequest = (url, realRoots = [], realPath = identityRealPath) => {
53
+ const parsed = parseUrl(url);
54
+ if (!parsed)
55
+ return false;
56
+ if (IN_MEMORY_SCHEMES.includes(parsed.protocol))
57
+ return true;
58
+ if (parsed.protocol === "file:")
59
+ return isInsideFileRoots(parsed, realRoots, realPath);
60
+ return isAllowedCdnUrl(parsed);
61
+ };
62
+ // A root that does not exist yet (the temp page before it is written) is its real parent plus its name.
63
+ export const realRoot = (root, realPath) => {
64
+ const resolved = nodePath.resolve(root);
65
+ const parent = nodePath.dirname(resolved);
66
+ return realPath(resolved) ?? nodePath.join(realPath(parent) ?? parent, nodePath.basename(resolved));
67
+ };
68
+ // Popups and preconnect hints open connections outside the page's request interception, so the whole browser
69
+ // talks through a proxy that does not exist, except for the CDNs (loopback included, which Chromium bypasses by default).
70
+ const UNREACHABLE_PROXY = "http://127.0.0.1:1";
71
+ export const strictNetworkLaunchArgs = () => [`--proxy-server=${UNREACHABLE_PROXY}`, `--proxy-bypass-list=${RENDER_CDN_HOSTS.join(";")};<-loopback>`];
72
+ // A popup the page writes into gets no evaluateOnNewDocument script, so it would still have WebRTC.
73
+ export const POPUP_BLOCK_SCRIPT = 'Object.defineProperty(globalThis, "open", { value: () => null, writable: false, configurable: false });';
74
+ // Request interception does not see WebSocket or EventSource connections; this does.
75
+ export const RENDER_CONTENT_SECURITY_POLICY = "connect-src 'self' data: blob:";
76
+ const CSP_META = `<meta http-equiv="Content-Security-Policy" content="${RENDER_CONTENT_SECURITY_POLICY}">`;
77
+ // The meta must come before anything a script could hide in, so it is never placed by searching for <head>
78
+ // (a comment or script text can contain one): right after a leading doctype, else first. Before a doctype it
79
+ // would switch the page to quirks mode; first otherwise, the parser still puts it in the head.
80
+ const LEADING_DOCTYPE = /^(\uFEFF?)(\s*<!doctype[^>]*>)?/i;
81
+ export const withRenderContentSecurityPolicy = (html) => {
82
+ const [prefix = "", bom = "", doctype] = LEADING_DOCTYPE.exec(html) ?? [];
83
+ const insertAt = doctype ? prefix.length : bom.length;
84
+ return [html.slice(0, insertAt), CSP_META, html.slice(insertAt)].join("");
85
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mulmocast",
3
- "version": "2.17.0",
3
+ "version": "2.19.0",
4
4
  "description": "",
5
5
  "type": "module",
6
6
  "main": "lib/index.node.js",
@@ -120,6 +120,7 @@
120
120
  "clipboardy": "^5.3.2",
121
121
  "dotenv": "^17.4.2",
122
122
  "graphai": "^2.0.18",
123
+ "jinglescript": "^0.6.0",
123
124
  "js-tiktoken": "^1.0.21",
124
125
  "jsdom": "^29.1.1",
125
126
  "marked": "^18.1.0",
@@ -149,7 +150,7 @@
149
150
  "@types/jsdom": "^28.0.3",
150
151
  "@types/node": "^26.6.4",
151
152
  "@types/yargs": "^17.0.35",
152
- "avatarscript": "^0.3.2",
153
+ "avatarscript": "^0.4.0",
153
154
  "cross-env": "^10.1.0",
154
155
  "esbuild": "^0.28.2",
155
156
  "eslint": "^10.12.0",
@@ -179,7 +180,7 @@
179
180
  "@remotion/shapes": "^4.0.533",
180
181
  "@remotion/three": "^4.0.533",
181
182
  "@remotion/transitions": "^4.0.533",
182
- "avatarscript": "^0.3.2",
183
+ "avatarscript": "^0.4.0",
183
184
  "onnxruntime-node": "^1.30.0",
184
185
  "react": "^19.2.0",
185
186
  "react-dom": "^19.2.0",
@@ -240,6 +241,6 @@
240
241
  }
241
242
  },
242
243
  "engines": {
243
- "node": ">=22.0.0"
244
+ "node": ">=22.13.0"
244
245
  }
245
246
  }
@@ -55,24 +55,25 @@
55
55
  "moviePrompt": "a woman takes a selfie with her phone"
56
56
  },
57
57
  {
58
- "id": "genai_veo3",
59
- "text": "movie generated by veo3",
58
+ "id": "genai_omni",
59
+ "text": "movie generated by gemini omni",
60
60
  "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
61
61
  "movieParams": {
62
- "model": "veo-3.1-lite-generate-preview"
62
+ "model": "gemini-omni-1.1-flash"
63
63
  }
64
64
  },
65
65
  {
66
- "id": "genai_veo3_1",
67
- "text": "movie generated by veo3_1",
66
+ "id": "genai_omni_long",
67
+ "text": "a twelve-second movie generated by gemini omni, extended once",
68
+ "duration": 12,
68
69
  "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
69
70
  "movieParams": {
70
- "model": "veo-3.1-generate-preview"
71
+ "model": "gemini-omni-1.1-flash"
71
72
  }
72
73
  },
73
74
  {
74
- "id": "genai_veo3_image",
75
- "text": "movie generated by veo3",
75
+ "id": "genai_omni_image",
76
+ "text": "movie generated by gemini omni from a first frame",
76
77
  "duration": 5,
77
78
  "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
78
79
  "imageParams": {
@@ -80,7 +81,7 @@
80
81
  },
81
82
  "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
82
83
  "movieParams": {
83
- "model": "veo-3.1-lite-generate-preview"
84
+ "model": "gemini-omni-1.1-flash"
84
85
  }
85
86
  }
86
87
  ]
@@ -11,7 +11,7 @@
11
11
  "lang": "en",
12
12
  "movieParams": {
13
13
  "provider": "google",
14
- "model": "veo-3.1-generate-preview"
14
+ "model": "gemini-omni-1.1-flash"
15
15
  },
16
16
  "beats": [
17
17
  {
@@ -15,7 +15,8 @@
15
15
  },
16
16
  "movieParams": {
17
17
  "provider": "google",
18
- "model": "veo-3.1-generate-preview"
18
+ "model": "veo-3.1-generate-001",
19
+ "vertexai_project": "YOUR_PROJECT_ID"
19
20
  },
20
21
  "lang": "en",
21
22
  "beats": [
@@ -24,7 +25,7 @@
24
25
  "text": "veo-3.1 with both ASSET and STYLE reference images",
25
26
  "moviePrompt": "the woman runs through narrow alleyways, neon signs blurring in the rain",
26
27
  "movieParams": {
27
- "model": "veo-3.1-generate-preview",
28
+ "model": "veo-3.1-generate-001",
28
29
  "referenceImages": [
29
30
  { "imageName": "character", "referenceType": "ASSET" },
30
31
  { "imageName": "black_cat", "referenceType": "ASSET" }
@@ -0,0 +1,101 @@
1
+ {
2
+ "$mulmocast": { "version": "1.1" },
3
+ "lang": "ja",
4
+ "title": "JingleScript sound effects test",
5
+ "audioParams": { "bgmVolume": 0.05 },
6
+ "beats": [
7
+ {
8
+ "text": "チク、タク、チク、タク。時計がなりますよ。",
9
+ "image": { "type": "textSlide", "slide": { "title": "チクタク、チーン" } },
10
+ "soundEffects": [
11
+ {
12
+ "source": {
13
+ "kind": "jinglescript",
14
+ "score": {
15
+ "format": "jinglescript/1",
16
+ "title": "Clock ticks into a chime",
17
+ "tempo": 120,
18
+ "length": { "seconds": 5 },
19
+ "cues": { "hit": { "seconds": 3 } },
20
+ "tracks": [
21
+ {
22
+ "instrument": "clock",
23
+ "reverb": false,
24
+ "gain": -4,
25
+ "notes": [{ "at": 0, "variant": ["tick", "tock"], "vel": 0.7, "repeat": { "every": 1, "until": "hit-1" } }]
26
+ },
27
+ {
28
+ "instrument": "glockenspiel",
29
+ "notes": [
30
+ { "at": "hit", "pitch": ["C6", "G6"], "vel": 0.8 },
31
+ { "at": "hit+1", "pitch": "C7", "vel": 0.4 }
32
+ ]
33
+ }
34
+ ]
35
+ }
36
+ },
37
+ "volume": 0.7
38
+ }
39
+ ]
40
+ },
41
+ {
42
+ "text": "ポン、ポン、ポン! 星が三つ出てきました。",
43
+ "image": { "type": "textSlide", "slide": { "title": "ポン、ポン、ポン" } },
44
+ "soundEffects": [
45
+ {
46
+ "source": {
47
+ "kind": "jinglescript",
48
+ "score": {
49
+ "format": "jinglescript/1",
50
+ "tempo": 120,
51
+ "length": { "seconds": 2 },
52
+ "tracks": [
53
+ {
54
+ "instrument": "pop",
55
+ "reverb": false,
56
+ "notes": [
57
+ { "at": 0, "pitch": "E6", "vel": 0.8 },
58
+ { "at": 0.5, "pitch": "G6", "vel": 0.8 },
59
+ { "at": 1, "pitch": "C7", "vel": 0.9 }
60
+ ]
61
+ }
62
+ ]
63
+ }
64
+ },
65
+ "startAt": 0.2
66
+ }
67
+ ]
68
+ },
69
+ {
70
+ "text": "よくできました!",
71
+ "image": { "type": "textSlide", "slide": { "title": "よくできました" } },
72
+ "soundEffects": [
73
+ {
74
+ "source": {
75
+ "kind": "jinglescript",
76
+ "score": {
77
+ "format": "jinglescript/1",
78
+ "title": "Well done",
79
+ "tempo": 130,
80
+ "length": { "seconds": 3 },
81
+ "cues": { "hit": { "seconds": 0.7 } },
82
+ "tracks": [
83
+ {
84
+ "instrument": "marimba",
85
+ "notes": [
86
+ { "at": 0, "pitch": "G4", "vel": 0.8 },
87
+ { "at": 0.5, "pitch": "C5", "vel": 0.85 },
88
+ { "at": "hit", "pitch": ["C5", "E5", "G5", "C6"], "vel": 1.0 }
89
+ ]
90
+ },
91
+ { "instrument": "marimba", "notes": [{ "at": "hit", "pitch": ["C3", "G3"], "vel": 0.9 }] }
92
+ ]
93
+ }
94
+ },
95
+ "startAt": 0.8,
96
+ "volume": 0.8
97
+ }
98
+ ]
99
+ }
100
+ ]
101
+ }
@@ -5,7 +5,8 @@
5
5
  "canvasSize": { "width": 1280, "height": 720 },
6
6
  "movieParams": {
7
7
  "provider": "google",
8
- "model": "veo-3.1-generate-preview"
8
+ "model": "veo-3.1-generate-001",
9
+ "vertexai_project": "YOUR_PROJECT_ID"
9
10
  },
10
11
  "imageParams": {
11
12
  "images": {
@@ -0,0 +1,64 @@
1
+ {
2
+ "$mulmocast": { "version": "1.1" },
3
+ "lang": "ja",
4
+ "title": "Sound effects test",
5
+ "audioParams": { "bgmVolume": 0.1 },
6
+ "beats": [
7
+ {
8
+ "text": "時計の針が動いています。チク、タク、チク、タク。",
9
+ "image": { "type": "textSlide", "slide": { "title": "チクタク" } },
10
+ "soundEffects": [
11
+ {
12
+ "source": {
13
+ "kind": "url",
14
+ "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/soundeffects/opengameart/bart-ticking-clock/ticking_clock.wav"
15
+ },
16
+ "startAt": 0,
17
+ "volume": 0.6,
18
+ "loop": true
19
+ }
20
+ ]
21
+ },
22
+ {
23
+ "text": "ボールがはねました。そして、コインをゲット!",
24
+ "image": { "type": "textSlide", "slide": { "title": "ビヨーン、チャリン" } },
25
+ "soundEffects": [
26
+ {
27
+ "source": {
28
+ "kind": "url",
29
+ "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/soundeffects/opengameart/ezduzziteh-boings/bing_01.mp3"
30
+ },
31
+ "startAt": 0.8
32
+ },
33
+ {
34
+ "source": {
35
+ "kind": "url",
36
+ "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/soundeffects/opengameart/junkala-512-retro-sfx/general-sounds/coins/sfx_coin_single1.wav"
37
+ },
38
+ "startAt": 2.2,
39
+ "volume": 0.7
40
+ }
41
+ ]
42
+ },
43
+ {
44
+ "text": "パチーン! 大正解です。",
45
+ "image": { "type": "textSlide", "slide": { "title": "大正解" } },
46
+ "soundEffects": [
47
+ {
48
+ "source": {
49
+ "kind": "url",
50
+ "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/soundeffects/kenney/impact-sounds/impactPunch_medium_000.ogg"
51
+ }
52
+ },
53
+ {
54
+ "source": {
55
+ "kind": "url",
56
+ "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/soundeffects/opengameart/junkala-512-retro-sfx/general-sounds/fanfares/sfx_sounds_fanfare1.wav"
57
+ },
58
+ "startAt": 0.6,
59
+ "volume": 0.8
60
+ }
61
+ ]
62
+ }
63
+ ]
64
+ }