@openvidstudio/mcp-server 0.1.0 → 1.4.0

package/dist/tools/musicCues.js

@@ -0,0 +1,252 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runMusicCues = runMusicCues;
+exports.registerMusicCues = registerMusicCues;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const zod_1 = require("zod");
+const util_1 = require("../util");
+const mcp_1 = require("./mcp");
+const SAMPLE_RATE = 22050;
+function cuesPathFor(trackAbs) {
+    const dir = path.dirname(trackAbs);
+    const stem = path.basename(trackAbs).replace(/\.[^.]+$/, "");
+    return path.join(dir, `${stem}.cues.json`);
+}
+/**
+ * Decodes to a temp raw-PCM file rather than piping through stdout: spawnCapture buffers
+ * stdout as a JS string (`chunk.toString()`), which is the right contract for every other
+ * caller (ffmpeg/ffprobe text output) but would corrupt raw s16le sample bytes. Writing to
+ * disk and reading the file back with fs (binary-safe) sidesteps that without changing a
+ * shared primitive several other tools depend on.
+ */
+async function decodeMonoPcm16(trackAbs, projectRoot) {
+    const tmpPath = path.join(path.dirname(trackAbs), `.${path.basename(trackAbs)}.${process.pid}.analyze.raw`);
+    try {
+        const res = await (0, util_1.spawnCapture)("ffmpeg", ["-v", "error", "-y", "-i", trackAbs, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", tmpPath], projectRoot);
+        if (res.code !== 0) {
+            throw new Error(`ffmpeg could not decode "${trackAbs}" for analysis:\n${res.stderr.slice(-800)}`);
+        }
+        const buf = fs.readFileSync(tmpPath);
+        if (buf.length < SAMPLE_RATE) {
+            throw new Error(`"${trackAbs}" decoded to less than one second of audio. Is the file real audio?`);
+        }
+        // Copy out of the file-backed buffer before the temp file is removed underneath it.
+        const copy = Buffer.from(buf);
+        return new Int16Array(copy.buffer, copy.byteOffset, Math.floor(copy.length / 2));
+    }
+    finally {
+        fs.rmSync(tmpPath, { force: true });
+    }
+}
+/** Short-window RMS envelope. Hop and window chosen for ~10-15ms resolution at 22050hz. */
+function rmsEnvelope(samples) {
+    const windowSize = 512;
+    const hopSize = 256;
+    const frames = Math.max(0, Math.floor((samples.length - windowSize) / hopSize) + 1);
+    const values = new Float64Array(frames);
+    for (let f = 0; f < frames; f++) {
+        const start = f * hopSize;
+        let sum = 0;
+        for (let i = 0; i < windowSize; i++) {
+            const s = samples[start + i] / 32768;
+            sum += s * s;
+        }
+        values[f] = Math.sqrt(sum / windowSize);
+    }
+    return { hopSeconds: hopSize / SAMPLE_RATE, values };
+}
+/** Positive first difference of the envelope: energy rising, not just energy present. */
+function onsetFlux(env) {
+    const flux = new Float64Array(env.length);
+    for (let i = 1; i < env.length; i++)
+        flux[i] = Math.max(0, env[i] - env[i - 1]);
+    return flux;
+}
+function mean(xs) {
+    let s = 0;
+    for (const x of xs)
+        s += x;
+    return xs.length ? s / xs.length : 0;
+}
+function stddev(xs, m) {
+    let s = 0;
+    for (const x of xs)
+        s += (x - m) * (x - m);
+    return xs.length ? Math.sqrt(s / xs.length) : 0;
+}
+/** Local maxima above threshold, with a refractory gap so one transient isn't double-picked. */
+function pickPeaks(flux, hopSeconds, minGapSeconds) {
+    const m = mean(flux);
+    const sd = stddev(flux, m);
+    const threshold = m + 1.3 * sd;
+    const minGapFrames = Math.max(1, Math.round(minGapSeconds / hopSeconds));
+    const peaks = [];
+    let lastPeak = -Infinity;
+    for (let i = 1; i < flux.length - 1; i++) {
+        if (flux[i] < threshold)
+            continue;
+        if (flux[i] < flux[i - 1] || flux[i] < flux[i + 1])
+            continue;
+        if (i - lastPeak < minGapFrames) {
+            if (peaks.length > 0 && flux[i] > peaks[peaks.length - 1].strength) {
+                peaks[peaks.length - 1] = { index: i, strength: flux[i] };
+                lastPeak = i;
+            }
+            continue;
+        }
+        peaks.push({ index: i, strength: flux[i] });
+        lastPeak = i;
+    }
+    return peaks;
+}
+/**
+ * Autocorrelate the onset envelope over a plausible tempo range to estimate BPM, then
+ * build an evenly-spaced beat grid at that tempo anchored to the first strong peak.
+ */
+function estimateTempoAndGrid(flux, hopSeconds, peaks, durationSeconds) {
+    const minBpm = 70;
+    const maxBpm = 180;
+    const minLag = Math.round(60 / maxBpm / hopSeconds);
+    const maxLag = Math.round(60 / minBpm / hopSeconds);
+    let bestLag = minLag;
+    let bestScore = -Infinity;
+    for (let lag = minLag; lag <= Math.min(maxLag, flux.length - 1); lag++) {
+        let score = 0;
+        for (let i = 0; i + lag < flux.length; i++)
+            score += flux[i] * flux[i + lag];
+        if (score > bestScore) {
+            bestScore = score;
+            bestLag = lag;
+        }
+    }
+    const beatSeconds = bestLag * hopSeconds;
+    const bpm = Math.round((60 / beatSeconds) * 10) / 10;
+    const anchor = peaks.length > 0 ? peaks[0].index * hopSeconds : 0;
+    const beats = [];
+    // Walk backward from the anchor to cover the track from t=0, then forward to the end.
+    let t = anchor;
+    while (t > 0)
+        t -= beatSeconds;
+    for (; t <= durationSeconds; t += beatSeconds) {
+        if (t >= 0)
+            beats.push(Number(t.toFixed(3)));
+    }
+    return { bpm, beats };
+}
+async function runMusicCues(input) {
+    const projectRoot = (0, util_1.resolveProjectRoot)(input.projectRoot);
+    (0, util_1.sanitizeRelativeOutPath)(projectRoot, input.track, "track");
+    const trackAbs = path.join(projectRoot, input.track);
+    if (!fs.existsSync(trackAbs)) {
+        throw new Error(`"${input.track}" does not exist under ${projectRoot}.`);
+    }
+    const cuesAbs = cuesPathFor(trackAbs);
+    const existing = fs.existsSync(cuesAbs)
+        ? JSON.parse(fs.readFileSync(cuesAbs, "utf8"))
+        : null;
+    // A "generated" cues file (written at synth time by compose_score) is exact by
+    // construction. force is for re-running the
+    // estimator against a track that was itself only ever estimated; it must never let a
+    // noisier re-analysis silently replace ground truth.
+    if (existing?.source === "generated") {
+        if (input.force) {
+            throw new Error(`${cuesAbs} is a generated cue file (exact by construction), not an estimated one. ` +
+                `force is for re-analyzing a track whose cues were themselves only ever estimated; ` +
+                `it refuses to overwrite generated ground truth.`);
+        }
+        return {
+            ...existing,
+            source: "cached",
+            cuesPath: path.relative(projectRoot, cuesAbs).replace(/\\/g, "/"),
+        };
+    }
+    if (!input.force && existing) {
+        return {
+            ...existing,
+            source: "cached",
+            cuesPath: path.relative(projectRoot, cuesAbs).replace(/\\/g, "/"),
+        };
+    }
+    const samples = await decodeMonoPcm16(trackAbs, projectRoot);
+    const durationSeconds = samples.length / SAMPLE_RATE;
+    const { hopSeconds, values: env } = rmsEnvelope(samples);
+    const flux = onsetFlux(env);
+    const peaks = pickPeaks(flux, hopSeconds, 0.15);
+    const { bpm, beats } = estimateTempoAndGrid(flux, hopSeconds, peaks, durationSeconds);
+    // strongCues: the loudest quarter of picked onsets, sorted back into time order. This is
+    // deliberately sparser than beats[], matching compose_score's strongCues-are-bar-downbeats
+    // shape: a handful of moments worth cutting on, not every detected transient.
+    const sortedByStrength = [...peaks].sort((a, b) => b.strength - a.strength);
+    const strongCount = Math.max(1, Math.round(peaks.length * 0.25));
+    const strongCues = sortedByStrength
+        .slice(0, strongCount)
+        .map((p) => Number((p.index * hopSeconds).toFixed(3)))
+        .sort((a, b) => a - b);
+    const result = {
+        track: input.track.replace(/\\/g, "/"),
+        bpm,
+        durationSeconds: Number(durationSeconds.toFixed(3)),
+        beats,
+        strongCues,
+        source: "analyzed",
+    };
+    fs.writeFileSync(cuesAbs, JSON.stringify(result, null, 2) + "\n", "utf8");
+    return { ...result, cuesPath: path.relative(projectRoot, cuesAbs).replace(/\\/g, "/") };
+}
+function registerMusicCues(server) {
+    server.registerTool("plan_music_cues", {
+        title: "Detect or read beat and strong-cue timestamps for a music track",
+        description: "Returns beats (the full beat grid) and strongCues (bar downbeats or the loudest onsets, a sparser " +
+            "set) for a music track, in seconds, so scene cuts can be beat-locked instead of landing at an " +
+            "arbitrary second. For a track compose_score wrote, this is a file read of the exact cue grid " +
+            "computed when the track was synthesized. For any other track it decodes the audio and estimates " +
+            "a grid from an RMS onset " +
+            "envelope and one autocorrelation pass: a real but simple detector, good enough to plan cuts against, " +
+            "not a substitute for listening to the track. Results are cached next to the track as " +
+            "<stem>.cues.json, so call again with force to re-analyze. force refuses to run against a generated " +
+            "cues file (compose_score's own): that grid is exact by construction, so force exists to redo an " +
+            "estimate, never to overwrite ground truth with a noisier one. See STYLE.md's \"Beat-locked cuts\" " +
+            "section for how to use the result: snap at most a handful of major reveals to strongCues within " +
+            "about 0.15s, snap sequential reveals (cards, stats arriving one by one) to consecutive beats within " +
+            "about 0.10s, and ignore a cue outright when honoring it would hurt readability or pacing.",
+        inputSchema: {
+            projectRoot: zod_1.z.string().optional(),
+            track: zod_1.z.string().min(1).describe('Path relative to projectRoot, e.g. "public/audio/music-bed.mp3".'),
+            force: zod_1.z.boolean().optional(),
+        },
+    }, async (input) => (0, mcp_1.runTool)("plan_music_cues", () => runMusicCues(input)));
+}

package/dist/tools/musicLibrary.d.ts

@@ -0,0 +1,80 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+declare const SOURCE_META: {
+    readonly lofi: {
+        readonly repo: "btahir/open-lofi";
+        readonly url: "https://github.com/btahir/open-lofi";
+        readonly licence: "CC0-1.0";
+    };
+    readonly cc0: {
+        readonly repo: "SoundSafari/CC0-1.0-Music";
+        readonly url: "https://github.com/SoundSafari/CC0-1.0-Music";
+        readonly licence: "CC0-1.0";
+    };
+};
+type SourceKey = keyof typeof SOURCE_META;
+export interface FindMusicBedInput {
+    projectRoot?: string;
+    query?: string;
+    source?: "lofi" | "cc0" | "any";
+    category?: string;
+    limit?: number;
+    refresh?: boolean;
+}
+export interface MusicCandidate {
+    id: string;
+    title: string;
+    source: SourceKey;
+    category: string;
+    megabytes: number;
+    licence: string;
+    sourceRepo: string;
+}
+export interface FindMusicBedResult {
+    query: string;
+    indexedAt: string;
+    trackCount: number;
+    candidates: MusicCandidate[];
+    categories: {
+        lofi: Record<string, string>;
+        cc0: string[];
+    };
+    licenceNote: string;
+    /** False when nothing matched the query and the candidates are a plain sample instead. */
+    matchedQuery: boolean;
+    nextStep: string;
+    /** Present only when source: "any" degraded: one catalog couldn't be fetched and the
+     * other's results are being returned alone (or a stale cache of the failed one). */
+    warnings?: string[];
+}
+export interface ImportMusicBedInput {
+    projectRoot?: string;
+    id: string;
+    out?: string;
+    seconds?: number;
+    start?: number;
+    fadeInSeconds?: number;
+    fadeOutSeconds?: number;
+    lufs?: number;
+    raw?: boolean;
+}
+export interface ImportMusicBedResult {
+    file: string;
+    durationSeconds: number | null;
+    bytes: number;
+    track: {
+        id: string;
+        title: string;
+        category: string;
+        sourceRepo: string;
+        licence: string;
+    };
+    processing: string;
+    provenanceFile: string;
+    ledgerFile: string;
+    licenceNote: string;
+    nextStep: string;
+}
+export declare function runFindMusicBed(input: FindMusicBedInput): Promise<FindMusicBedResult>;
+export declare function runImportMusicBed(input: ImportMusicBedInput): Promise<ImportMusicBedResult>;
+export declare function registerMusicLibrary(server: McpServer): void;
+export {};