@openvidstudio/mcp-server 0.1.0 → 1.4.0

package/dist/tools/soundEffects.js

@@ -0,0 +1,383 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runSoundEffects = runSoundEffects;
+exports.registerSoundEffects = registerSoundEffects;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const zod_1 = require("zod");
+const util_1 = require("../util");
+const mcp_1 = require("./mcp");
+/**
+ * Sourcing sound effects that are not in the synthesized pack.
+ *
+ * Most of the built-in pack is generated by ffmpeg from anoisesrc and sine. The rest is
+ * typing-effect.mp3, a copyright-free recording, and bong/drop/click_soft, CC0 material
+ * from Kenney; public/sfx/CREDITS.md lists both. Either way the pack carries no rights
+ * anyone has to honor:
+ * no attribution, no content ID, no licence to breach. That is a real property of this
+ * project and the reason the default here is "use what is already there". Every sound
+ * you add from outside gives that up, and CC-BY material has to be credited in the
+ * video description or you are in breach of the licence.
+ *
+ * On Pixabay specifically. Their Terms of Service prohibit "data mining, extraction,
+ * scraping and the use of programs or robots for automatic data collection", and state
+ * that the API "is made for real human requests" with "systematic mass downloads" not
+ * allowed. Their public API also covers images and video only; there is no documented
+ * sound-effects endpoint. So a tool that downloads from Pixabay on the user's behalf
+ * would have to scrape, and shipping that in an open-source package would put every
+ * install in breach. What this tool does instead is prepare the search: it works out
+ * which sounds are missing, hands back one Pixabay search link per gap and the exact
+ * path to save each file to, and then verifies and normalizes whatever the user
+ * actually downloaded. The human does the download, which is the part Pixabay's terms
+ * are written around.
+ *
+ * Freesound is offered as the automated path because it has a real public API built
+ * for programmatic access. It needs the user's own token and it records the licence
+ * and attribution for every clip it fetches.
+ */
+/** The pack that ships with @openvidstudio/core: synthesized, a copyright-free typing recording, three CC0 sounds and a Mixkit click (see public/sfx/CREDITS.md). Using it in a video needs no credit. */
+const BUILT_IN = {
+    bell: "soft notification chime, harmonic stack",
+    blip: "short UI beep",
+    bong: "warm soft-mallet accent, short decay",
+    click: "mouse or UI click",
+    click_soft: "quieter, rounder click than click.wav",
+    drop: "low soft thud for something landing or settling",
+    "typing-effect": "continuous typing, one 8s copyright-free recording that plays across a typed run",
+    success: "positive confirmation tone",
+    whoosh: "transition sweep",
+    "music-bed": "ambient background bed, detuned sines plus pink noise",
+};
+/**
+ * Maps a plain-English request onto the built-in pack, so the tool can say "you already
+ * have this" instead of sending someone to download a click sound that ships in the box.
+ */
+const SYNONYMS = [
+    [/\b(soft|quiet|subtle|gentle) (click|tap|press)\b/i, "click_soft"],
+    [/\b(click|tap|press|button)\b/i, "click"],
+    [/\b(bong|mallet|marimba)\b/i, "bong"],
+    [/\b(drop|thud|land|landing|settle)\b/i, "drop"],
+    [/\b(key|keystroke|typing|type|keyboard|enter|return|submit)\b/i, "typing-effect"],
+    [/\b(success|complete|done|confirm|finish)\b/i, "success"],
+    [/\b(whoosh|swoosh|transition|sweep|swipe)\b/i, "whoosh"],
+    [/\b(bell|chime|notification|ding|alert)\b/i, "bell"],
+    [/\b(blip|beep|bloop|tick)\b/i, "blip"],
+    [/\b(music|bed|ambient|background|underscore|track)\b/i, "music-bed"],
+];
+const AUDIO_EXT = [".mp3", ".wav", ".ogg", ".m4a", ".flac"];
+function matchBuiltIn(description, id) {
+    const hay = `${id} ${description}`;
+    if (BUILT_IN[id])
+        return id;
+    for (const [re, name] of SYNONYMS)
+        if (re.test(hay))
+            return name;
+    return null;
+}
+function probeDuration(file) {
+    const r = (0, node_child_process_1.spawnSync)("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", file], { encoding: "utf8" });
+    const d = Number.parseFloat((r.stdout ?? "").trim());
+    return Number.isFinite(d) && d > 0 ? d : null;
+}
+function findExisting(importedAudiosDir, id) {
+    for (const ext of AUDIO_EXT) {
+        const p = path.join(importedAudiosDir, `${id}${ext}`);
+        if (fs.existsSync(p))
+            return p;
+    }
+    return null;
+}
+/** Pixabay's own search page for this sound. The user downloads from here by hand. */
+function pixabaySearchUrl(description) {
+    return `https://pixabay.com/sound-effects/search/${encodeURIComponent(description.trim())}/`;
+}
+async function freesoundSearch(query, token) {
+    const url = "https://freesound.org/apiv2/search/text/?" +
+        new URLSearchParams({
+            query,
+            fields: "id,name,username,license,previews",
+            page_size: "1",
+            token,
+        }).toString();
+    const res = await fetch(url);
+    if (!res.ok) {
+        throw new Error(`Freesound returned ${res.status}. A 401 means the token is wrong; a 429 means you are over the rate limit.`);
+    }
+    const body = (await res.json());
+    const hit = body.results?.[0];
+    if (!hit)
+        return null;
+    const preview = hit.previews?.["preview-hq-mp3"] ?? hit.previews?.["preview-lq-mp3"];
+    if (!preview)
+        return null;
+    return {
+        id: hit.id,
+        name: hit.name,
+        username: hit.username,
+        license: hit.license,
+        previewUrl: preview,
+    };
+}
+async function download(url, dest) {
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`download failed with ${res.status} for ${url}`);
+    const buf = Buffer.from(await res.arrayBuffer());
+    if (buf.byteLength < 1024)
+        throw new Error(`downloaded file is too small to be audio: ${dest}`);
+    fs.writeFileSync(dest, buf);
+}
+/** Re-encode to one format so scenes are not guessing at extensions. */
+function normalize(file, to) {
+    const target = file.replace(/\.[^.]+$/, `.${to}`);
+    if (target === file)
+        return file;
+    const args = to === "mp3"
+        ? ["-v", "error", "-i", file, "-c:a", "libmp3lame", "-b:a", "160k", target, "-y"]
+        : ["-v", "error", "-i", file, "-c:a", "pcm_s16le", "-ar", "44100", target, "-y"];
+    const r = (0, node_child_process_1.spawnSync)("ffmpeg", args, { encoding: "utf8" });
+    if (r.status !== 0)
+        return file;
+    fs.rmSync(file, { force: true });
+    return target;
+}
+/**
+ * The checklist the user actually works from. It goes on disk next to the folder they
+ * are dropping files into, because a list that only exists in a chat transcript is lost
+ * the moment the conversation scrolls or the session ends.
+ */
+function writeDownloadManifest(importedAudiosDir, rows) {
+    const wanted = rows.filter((r) => r.status === "needs-download");
+    const out = path.join(importedAudiosDir, "DOWNLOAD-THESE.md");
+    if (wanted.length === 0) {
+        fs.rmSync(out, { force: true });
+        return undefined;
+    }
+    const lines = [
+        "# Sounds to download",
+        "",
+        `Save every file into this folder: \`${importedAudiosDir}\``,
+        "",
+        "Use the exact filename in the Save as column. The scenes look for that name, so a",
+        "file called `whoosh (2).mp3` will not be found.",
+        "",
+        "Any common audio format works. Once the files are here, ask your agent to run",
+        "`plan_sound_effects` again with `verify: true` and it will check each one is real",
+        "audio and convert the formats.",
+        "",
+        "| # | Sound | Save as | Search |",
+        "|---|---|---|---|",
+        ...wanted.map((r, i) => `| ${i + 1} | ${r.description} | \`${path.basename(r.savePath ?? r.id)}\` | [Pixabay](${r.searchUrl}) |`),
+        "",
+        "## Before you download",
+        "",
+        "Pixabay sounds are covered by the Pixabay Content Licence. Check the licence shown",
+        "on each sound's page, because a handful of contributors ask for credit. Anything",
+        "needing credit belongs in the video description.",
+        "",
+        "Anything else already in this folder was added in an earlier round; check",
+        "`ATTRIBUTION.md` next to this file for its licence and credit.",
+        "",
+    ];
+    fs.writeFileSync(out, lines.join("\n"), "utf8");
+    return out;
+}
+function writeAttribution(importedAudiosDir, rows) {
+    const credited = rows.filter((r) => r.attribution);
+    if (credited.length === 0)
+        return undefined;
+    const out = path.join(importedAudiosDir, "ATTRIBUTION.md");
+    const lines = [
+        "# Sound effect attribution",
+        "",
+        "These clips came from outside the synthesized pack and carry licence terms.",
+        "Reproduce this list in the video description. Removing it can breach the licence.",
+        "",
+        "| File | Source | Author | Licence |",
+        "|---|---|---|---|",
+        ...credited.map((r) => `| ${path.basename(r.file ?? r.id)} | ${r.attribution} | | ${r.license ?? "see source"} |`),
+        "",
+        "Everything else in this directory was dropped in by hand; check with whoever added it",
+        "for its source and licence before reusing this video's audio elsewhere.",
+        "",
+    ];
+    fs.writeFileSync(out, lines.join("\n"), "utf8");
+    return out;
+}
+async function runSoundEffects(input) {
+    const root = (0, util_1.resolveProjectRoot)(input.projectRoot);
+    const importedAudiosDir = path.join(root, "public", "imported_audios");
+    fs.mkdirSync(importedAudiosDir, { recursive: true });
+    const provider = input.provider ?? "manual";
+    const token = input.freesoundToken ?? process.env.FREESOUND_API_KEY ?? "";
+    if (provider === "freesound" && !token) {
+        throw new Error("Freesound needs a token. Create a free account, make an API key at " +
+            "freesound.org/apiv2/apply, then pass freesoundToken or set FREESOUND_API_KEY.");
+    }
+    const rows = [];
+    for (const need of input.needs) {
+        const id = (0, util_1.sanitizeSegment)(need.id, "sound effect id");
+        const builtIn = matchBuiltIn(need.description, id);
+        if (builtIn) {
+            rows.push({
+                id,
+                description: need.description,
+                status: "already-built-in",
+                builtInName: builtIn,
+                file: `sfx/${builtIn}${builtIn === "music-bed" ? ".mp3" : ".wav"}`,
+            });
+            continue;
+        }
+        const existing = findExisting(importedAudiosDir, id);
+        if (existing) {
+            const file = input.verify && input.normalizeTo ? normalize(existing, input.normalizeTo) : existing;
+            rows.push({
+                id,
+                description: need.description,
+                status: "present",
+                file: path.relative(root, file).replace(/\\/g, "/"),
+                durationSec: probeDuration(file) ?? undefined,
+            });
+            continue;
+        }
+        if (provider === "freesound" && !input.verify) {
+            const hit = await freesoundSearch(need.description, token);
+            if (hit) {
+                const dest = path.join(importedAudiosDir, `${id}.mp3`);
+                await download(hit.previewUrl, dest);
+                rows.push({
+                    id,
+                    description: need.description,
+                    status: "present",
+                    file: path.relative(root, dest).replace(/\\/g, "/"),
+                    durationSec: probeDuration(dest) ?? undefined,
+                    license: hit.license,
+                    attribution: `freesound.org/s/${hit.id} by ${hit.username}`,
+                });
+                continue;
+            }
+        }
+        rows.push({
+            id,
+            description: need.description,
+            status: "needs-download",
+            searchUrl: pixabaySearchUrl(need.description),
+            savePath: path.join(importedAudiosDir, `${id}.mp3`),
+        });
+    }
+    const missing = rows.filter((r) => r.status === "needs-download");
+    const outstanding = missing.length;
+    const covered = rows.length - outstanding;
+    const attributionFile = writeAttribution(importedAudiosDir, rows);
+    const manifestFile = writeDownloadManifest(importedAudiosDir, rows);
+    const instructions = outstanding === 0
+        ? [
+            `Every sound is in place. Built-ins live in public/sfx/; anything imported lives in ` +
+                `${importedAudiosDir} and scenes reference it as imported_audios/<id>.`,
+        ]
+        : [
+            `${outstanding} sound${outstanding === 1 ? "" : "s"} still needed. ` +
+                `Download each one and drop it into this folder:`,
+            importedAudiosDir,
+            "",
+            ...missing.map((r, i) => `${i + 1}. ${r.description}\n` +
+                `   search:  ${r.searchUrl}\n` +
+                `   save as: ${path.basename(r.savePath ?? r.id)}`),
+            "",
+            "The filename matters, the scenes look for it exactly. Any common audio format is fine.",
+            manifestFile ? `Same list is saved at ${manifestFile}.` : "",
+            "Tell me once they are in and I will verify them.",
+        ].filter(Boolean);
+    const nextStep = outstanding === 0
+        ? "Every sound is accounted for. Reference imported ones from scenes as imported_audios/<id>."
+        : `Give the user the instructions list, wait for them to confirm the files are in ` +
+            `${importedAudiosDir}, then call this tool again with verify: true to check each one is real ` +
+            `audio and normalize the formats.`;
+    return {
+        provider,
+        importedAudiosDir,
+        needs: rows,
+        covered,
+        outstanding,
+        attributionFile,
+        manifestFile,
+        instructions,
+        licenceNote: "The built-in pack in public/sfx/ is synthesized by ffmpeg, apart from a copyright-free typing " +
+            "recording, three CC0 sounds from Kenney and one Mixkit click, all recorded in public/sfx/CREDITS.md. " +
+            "Using any of it in a video asks for no credit. " +
+            "Anything imported into public/imported_audios/ does carry terms: Pixabay content is covered by " +
+            "their Content Licence, and Freesound clips are per-clip CC and often require credit. Whatever " +
+            "lands in ATTRIBUTION.md belongs in the video description.",
+        nextStep,
+    };
+}
+function registerSoundEffects(server) {
+    server.registerTool("plan_sound_effects", {
+        title: "Work out which sound effects are needed and where to get them",
+        description: "Ask the user what sounds the video needs, then call this with one entry per sound. It first " +
+            "checks the pack that ships with the project, which already covers clicks (two weights), " +
+            "keystrokes, a bell, a blip, a bong accent, a low drop, a success tone, a whoosh and an ambient " +
+            "music bed (for a score that follows the video's arc, see compose_score and plan_music_cues), so " +
+            "people are not sent off to download a click sound that is already in the box. For anything genuinely missing it " +
+            "returns an 'instructions' list: one Pixabay search link per sound and the exact filename to " +
+            "save it as, all going into public/imported_audios/, kept separate from the built-in pack in " +
+            "public/sfx/ so it is always clear which sounds carry licence terms. Read that list to the user " +
+            "verbatim and wait for them to say the files are in place; the same checklist is written to " +
+            "DOWNLOAD-THESE.md inside the folder so it survives the conversation. Then call this again with " +
+            "verify true, which confirms " +
+            "each file is real audio and normalizes the formats. This tool does not download from Pixabay " +
+            "and neither should you: their terms prohibit automated collection and state the API is for " +
+            "real human requests, and they publish no sound-effects endpoint, so the user does that step. " +
+            "Set provider to 'freesound' to fetch automatically instead, which needs the user's own API " +
+            "token and records licence and attribution per clip. Note that the built-in pack asks " +
+            "for no credit, and every downloaded sound gives that up, so prefer the " +
+            "built-ins when they fit.",
+        inputSchema: {
+            projectRoot: zod_1.z.string().optional(),
+            needs: zod_1.z
+                .array(zod_1.z.object({
+                id: zod_1.z.string().min(1),
+                description: zod_1.z.string().min(1),
+            }))
+                .min(1),
+            provider: zod_1.z.enum(["manual", "freesound"]).optional(),
+            verify: zod_1.z.boolean().optional(),
+            freesoundToken: zod_1.z.string().optional(),
+            normalizeTo: zod_1.z.enum(["mp3", "wav"]).optional(),
+        },
+    }, async (input) => (0, mcp_1.runTool)("plan_sound_effects", () => runSoundEffects(input)));
+}

package/dist/tools/stitchComposition.d.ts

@@ -2,6 +2,38 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 export interface StitchCompositionInput {
     projectRoot?: string;
     videoName: string;
+    /**
+     * Fail instead of silently shipping a partly-silent video. A missing VO file has
+     * always been survivable-by-design, but "survivable" quietly became "invisible":
+     * the Audio layer is skipped with no error and the render succeeds, so the first
+     * time anyone notices is on playback. Presets that declare narration mandatory
+     * (2min-demo, 5min-demo) should set this.
+     */
+    requireNarration?: boolean;
+    /**
+     * Emit one composition per language, reading each language's narration from its own
+     * directory. Defaults to the base language alone, which is every existing project.
+     */
+    languages?: string[];
+    /** The language public/audio/vo/<beatId>.mp3 is in. Defaults to en. */
+    baseLanguage?: string;
+    /**
+     * Overrides the auto-detected music bed and its mix level. `file` is a filename under
+     * public/audio/ (same convention as the auto-detected music-bed.mp3, not an arbitrary
+     * path). `volume` defaults to 0.18 (today's under-narration level) when
+     * beats.json's narration is "voice" or omitted, and 1.0 (full level, there is no VO to
+     * duck under) when narration is "none".
+     */
+    musicBed?: {
+        file: string;
+        volume?: number;
+    };
+}
+export interface LanguageComposition {
+    language: string;
+    compositionId: string;
+    voBeatsFound: string[];
+    voBeatsMissing: string[];
 }
 export interface StitchCompositionResult {
     written: true;
@@ -10,15 +42,82 @@ export interface StitchCompositionResult {
     rootPath: string;
     durationInFrames: number;
     voBeatsFound: string[];
+    /** Beats that will render silent. Previously knowable only by diffing two lists by hand. */
+    voBeatsMissing: string[];
     musicBedFound: boolean;
+    /**
+     * Which bed actually got used, when one did: the auto-detected "music-bed.mp3" (the
+     * built-in ambient bed, or the file compose_score writes by default), a track picked
+     * with musicBed, or a hand-picked layer carried over from the previous stitch.
+     */
+    musicBedFile: string | null;
+    /** The mix level actually applied to musicBedFile, honest about the default that was picked. */
+    musicBedVolume: number;
+    /**
+     * Where the music layer came from: "explicit" (musicBed input), "preserved" (the track and
+     * level the previous stitch of this video had, so a hand-picked bed survives a re-stitch),
+     * "auto" (music-bed.mp3 found in public/audio), or "none".
+     */
+    musicBedSource: "explicit" | "preserved" | "auto" | "none";
+    /**
+     * Where the bed's cue grid was copied to for useMusicClock, relative to the project root, or
+     * null when the bed has no <stem>.cues.json next to it (then useMusicClock reports
+     * available: false and runs on a fallback grid). Run plan_music_cues to produce one.
+     */
+    musicCuesPath: string | null;
+    /** True when src/videos/<videoName>/overrides.tsx exists and was wired into the composition. */
+    overridesApplied: boolean;
+    /** Plain-language notices for things that succeeded but probably are not what was wanted. */
+    warnings: string[];
+    /** One entry per language, including the base. */
+    languages: LanguageComposition[];
+}
+/**
+ * The registry videoName init_project seeds preflight's benchmark composition under
+ * (preflight.ts) -- never a real video's name, so this entry survives every real video's
+ * stitch_composition call (whose registry filter is `entries.filter(e => e.videoName !==
+ * thisVideoName)`) untouched.
+ */
+export declare const PREFLIGHT_BENCHMARK_VIDEO_NAME = "__preflight__";
+/** The registry entry's own composition id, what preflight's benchmark actually renders. */
+export declare const PREFLIGHT_BENCHMARK_COMPOSITION_ID = "OvsPreflightBenchmark";
+export interface RegistryEntry {
+    id: string;
+    videoName: string;
+    componentName: string;
+    importPath: string;
+    durationInFrames: number;
+    fps: number;
+    width: number;
+    height: number;
 }
 /**
- * Reads src/videos/<videoName>/beats.json and writes the composition + updates
- * src/Root.tsx (via the registry above). VO convention: public/audio/vo/<beatId>.mp3,
- * included only if the file actually exists on disk (mirrors PepirosDemo.tsx, where
- * the cta beat has no VO entry because none was recorded). Music bed convention:
- * public/audio/music-bed.mp3, included as a full-duration looping layer only if it
- * exists.
+ * Registry of every video this project has stitched, one entry per stitch_composition
+ * call, at src/videos/.registry.json. Root.tsx is fully regenerated from this file on
+ * every call rather than round-tripped via regex/JSX parsing, which is what makes it
+ * safe to never clobber a composition a prior call registered for a different video:
+ * the registry (not Root.tsx itself) is the actual source of truth. Deliberately not
+ * part of openvidstudio.config.json: that shape is pinned for Task 5 to depend on,
+ * this is internal bookkeeping local to this one tool.
+ */
+export declare function registryPath(projectRoot: string): string;
+export declare function readRegistry(projectRoot: string): RegistryEntry[];
+export declare function writeRegistry(projectRoot: string, entries: RegistryEntry[]): void;
+export declare function renderRootTsx(entries: RegistryEntry[], hasBrand: boolean): string;
+/**
+ * Frames either side of a cut that a transition overlay covers: about 0.2s, so the whole
+ * overlay is 2 * this. Scaled with fps so the transition keeps its duration in seconds.
+ */
+export declare function transitionHalfFrames(fps: number): number;
+/**
+ * The music layer a previous stitch of this video wrote, if any. Re-stitching regenerates the
+ * whole composition file, so without this a hand-picked track or level silently reverts to the
+ * auto-detected bed. Matches only the exact shape renderDemoTsx emits.
  */
+export declare function readExistingMusicLayer(demoPath: string): {
+    file: string;
+    volume: number;
+    pinned: boolean;
+} | null;
 export declare function runStitchComposition(input: StitchCompositionInput): StitchCompositionResult;
 export declare function registerStitchComposition(server: McpServer): void;