creator-editing-studio 1.3.6 → 1.3.7

package/template/preview/server.mjs

@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+/**
+ * The live preview inside a Claude chat (an MCP App) — part of the studio.
+ *
+ * WHY THE PAGE CARRIES EVERYTHING. In the Claude desktop app the in-chat player cannot call this server
+ * back ("IPC not available for MCP tool call", tested 2026-10-17) and cannot fetch from the computer
+ * ("Failed to fetch", 2026-10-16). What it can do: show the page this server hands over when the preview
+ * opens, and send messages into the chat. So each time the preview opens, the page arrives with the
+ * compiled edit and small preview copies of the footage already inside it.
+ *
+ * THE LOOP. The person types a change under a beat → the player sends it into the chat → Claude edits
+ * that beat → Claude calls preview_edit again → the updated preview opens below. Every open saves a copy
+ * of the video's folder first, so preview_undo can put the last change back exactly.
+ *
+ * Which video: preview_edit({video}) — or current.json (written by use.mjs) when it is called without one. Nothing listens on any port. stdout belongs to the MCP
+ * protocol: every log goes to stderr.
+ */
+import {cpSync, existsSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs';
+import {join} from 'node:path';
+import {McpServer} from '@modelcontextprotocol/sdk/server/mcp.js';
+import {StdioServerTransport} from '@modelcontextprotocol/sdk/server/stdio.js';
+import {registerAppResource, registerAppTool, RESOURCE_MIME_TYPE} from '@modelcontextprotocol/ext-apps/server';
+import {z} from 'zod';
+import {buildManifest, compileEdit, configFor, HERE, playerHtml, SRC} from './lib.mjs';
+
+const CONFIG = join(HERE, 'current.json');
+const cfg = () => JSON.parse(readFileSync(CONFIG, 'utf8'));
+const log = (...a) => console.error('[live-preview]', ...a);
+const SNAPS = (project) => join(HERE, '.snapshots', project);
+const videoDir = (project) => join(SRC, 'videos', project);
+
+/* ----------------------------------------------------------------------------- versions (for Undo) */
+
+/** A fingerprint of the video's folder, so opening the preview twice without a change saves nothing. */
+const fingerprint = (dir) =>
+  readdirSync(dir, {recursive: true})
+    .map(String)
+    .sort()
+    .map((f) => {
+      const st = statSync(join(dir, f));
+      return st.isFile() ? `${f}:${st.size}:${st.mtimeMs}` : '';
+    })
+    .join('|');
+const sigFile = (project, v) => join(SNAPS(project), String(v), '.sig');
+
+const versions = (project) =>
+  existsSync(SNAPS(project))
+    ? readdirSync(SNAPS(project))
+        .map(Number)
+        .filter((v) => Number.isFinite(v) && existsSync(sigFile(project, v)))
+        .sort((a, b) => a - b)
+    : [];
+
+/** Save the video's folder as the next version if it changed since the last one. Returns the current version. */
+function saveVersion(project) {
+  const dir = videoDir(project);
+  const list = versions(project);
+  const last = list.at(-1);
+  const sig = fingerprint(dir);
+  if (last !== undefined && readFileSync(sigFile(project, last), 'utf8') === sig) return last;
+  const v = (last ?? 0) + 1;
+  cpSync(dir, join(SNAPS(project), String(v)), {recursive: true});
+  // The fingerprint of the live folder as it was saved (the copy's own file dates differ).
+  writeFileSync(sigFile(project, v), sig);
+  for (const old of list.slice(0, Math.max(0, list.length - 40))) rmSync(join(SNAPS(project), String(old)), {recursive: true, force: true});
+  return v;
+}
+
+/* ----------------------------------------------------------------------------- the page */
+
+let uiHtml = '';
+let uiSig = '';
+const media = new Map(); // hash → data URL, so unchanged footage is encoded once
+
+/** The player page with the current edit and its footage inside it. */
+async function page() {
+  const c = cfg();
+  const UI = join(HERE, 'ui');
+  const sig = existsSync(UI) ? readdirSync(UI).map((f) => `${f}:${statSync(join(UI, f)).mtimeMs}`).join('|') : 'built';
+  if (sig !== uiSig) [uiHtml, uiSig] = [await playerHtml(), sig];
+
+  const version = saveVersion(c.project);
+  let code = null;
+  let error = null;
+  try {
+    code = await compileEdit(c);
+  } catch (e) {
+    error = String(e.message ?? e);
+  }
+  const files = {};
+  for (const a of buildManifest(c)) {
+    if (!media.has(a.hash)) {
+      // Remotion's font loader reads the type from the text after the last dot, so fonts get it as a fragment.
+      const ext = /\.(ttf|otf|woff2?)$/i.exec(a.name)?.[1];
+      media.set(a.hash, `data:${a.mime};base64,${readFileSync(a.file).toString('base64')}${ext ? `#.${ext}` : ''}`);
+    }
+    files[a.name] = media.get(a.hash);
+  }
+  const data = {version, canUndo: versions(c.project).length > 1, project: c.project, edit: c.edit, composition: c.composition, error, code, files};
+  const html = uiHtml.replace('<head>', `<head><script>window.__LP_DATA=${JSON.stringify(data).replace(/</g, '\\u003c')}</script>`);
+  log(`page for ${c.project} version ${version}: ${(html.length / 1e6).toFixed(1)} MB${error ? ' (the edit does not compile)' : ''}`);
+  return html;
+}
+
+/* ----------------------------------------------------------------------------- MCP */
+
+const URI = 'ui://editing-os/live-preview.html';
+const server = new McpServer({name: 'editing-os-live-preview', version: '0.1.0'});
+
+registerAppTool(
+  server,
+  'preview_edit',
+  {
+    title: 'Live preview',
+    description:
+      'Show the live preview of the current edit inside the chat: the video plays with the beats beside it, and a change box under every beat. Nothing is rendered. ' +
+      'OPEN IT YOURSELF as a step of the work — right after the person approves the plan, and again after every change you make to the edit — never wait to be asked. ' +
+      'Changes typed in the preview arrive as chat messages: change only that beat in the edit file, then call preview_edit again so the updated preview appears. ' +
+      'Render the final video only when they press "Render final video", and give the estimated render time only then.',
+    inputSchema: {video: z.string().optional().describe('The video folder name (src/videos/<video>). Needed the first time for each video.')},
+    _meta: {ui: {resourceUri: URI}},
+  },
+  async ({video}) => {
+    if (video) writeFileSync(CONFIG, JSON.stringify(configFor(video), null, 2) + '\n');
+    if (!existsSync(CONFIG)) return {content: [{type: 'text', text: 'Which video? Call preview_edit with {video: "<folder name in src/videos>"}.'}], isError: true};
+    const c = cfg();
+    const v = saveVersion(c.project);
+    return {content: [{type: 'text', text: `Live preview of ${c.project} (version ${v}) is showing in the chat.`}], structuredContent: {project: c.project, version: v}};
+  },
+);
+
+server.registerTool(
+  'preview_undo',
+  {
+    title: 'Undo the last preview change',
+    description: 'Put the edit back exactly as it was before the last change. Use it when the person presses Undo in the preview, then call preview_edit to show it.',
+  },
+  async () => {
+    const c = cfg();
+    const list = versions(c.project);
+    if (list.length < 2) return {content: [{type: 'text', text: 'Nothing to undo — this is the first version of the edit.'}], isError: true};
+    const [prev, last] = list.slice(-2);
+    const dir = videoDir(c.project);
+    const from = join(SNAPS(c.project), String(prev));
+    for (const f of readdirSync(dir)) rmSync(join(dir, f), {recursive: true, force: true});
+    cpSync(from, dir, {recursive: true});
+    rmSync(join(dir, '.sig'), {force: true});
+    rmSync(join(SNAPS(c.project), String(last)), {recursive: true, force: true});
+    // The restored folder is version `prev` again; remember its new fingerprint so it is not saved twice.
+    writeFileSync(sigFile(c.project, prev), fingerprint(dir));
+    log(`undo: ${c.project} back to version ${prev}`);
+    return {content: [{type: 'text', text: `Undone — ${c.project} is back to version ${prev}. Call preview_edit to show it.`}]};
+  },
+);
+
+registerAppResource(server, 'Live preview', URI, {description: 'Player and beat breakdown for the current edit'}, async () => ({
+  contents: [{uri: URI, mimeType: RESOURCE_MIME_TYPE, text: await page(), _meta: {ui: {prefersBorder: false}}}],
+}));
+
+await server.connect(new StdioServerTransport());
+log('MCP server ready');

package/template/preview/use.mjs

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+/**
+ * Point the live preview at a video (preview_edit({video}) does the same).
+ *
+ *   node preview/use.mjs <name>
+ *
+ * Writes current.json from the video's own folder (see configFor in lib.mjs); the running preview server
+ * watches that file and switches over by itself. The online preview uses publish.mjs instead.
+ */
+import {writeFileSync} from 'node:fs';
+import {join} from 'node:path';
+import {configFor, HERE} from './lib.mjs';
+
+try {
+  const config = configFor(process.argv[2]);
+  if (!config.stems.voice) console.warn('warning: no voice stem found — the preview will play silent');
+  writeFileSync(join(HERE, 'current.json'), JSON.stringify(config, null, 2) + '\n');
+  console.log(JSON.stringify(config, null, 2));
+} catch (e) {
+  console.error(String(e.message ?? e));
+  process.exit(1);
+}

package/template/requirements.txt

@@ -21,3 +21,7 @@ torch>=2.2
 numpy>=1.26
 Pillow>=10.0
 certifi>=2024.2.2
+
+# Motion tracking (scripts/track_head.py): optical-flow tracking of the speaker's head, so text can ride
+# with it. Headless build: no GUI libraries, works on servers and Macs alike. Only used when asked for.
+opencv-python-headless>=4.9

package/template/scripts/align-script.py

@@ -0,0 +1,138 @@
+"""Word timings from a script you already have: match each word of an .srt to the audio.
+
+Usage (from the project root):
+    python scripts/align-script.py <media> <script.srt> --language ml [--project name]
+
+Writes work/<project>/<clip>/words.json in the same shape as transcribe.py, so everything downstream
+works unchanged.
+
+WHY THIS EXISTS. For Indian languages, speech recognition is the weak link: on a Malayalam take the
+small Whisper model ignored the speech and invented English, and even large models misspell names and
+loanwords. When the creator already has the script (or a corrected .srt), the words are known — only
+their timing is not. Forced alignment (wav2vec2, per language) answers exactly that, and it is far more
+reliable than recognition: it cannot invent words, it only places the ones it is given.
+
+The .srt timings only need to be roughly right — each line is aligned inside its own window, padded a
+little either side, so a line timed a few hundred milliseconds off still lands correctly.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys as _sys
+import time
+from pathlib import Path
+
+_sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
+from media import FFMPEG, use_certifi  # noqa: E402
+from languages import lang, load_aligner, studio_language  # noqa: E402
+
+_MODELS = Path(__file__).resolve().parent.parent / ".models"
+os.environ.setdefault("HF_HOME", str(_MODELS / "huggingface"))
+os.environ.setdefault("TORCH_HOME", str(_MODELS / "torch"))
+os.environ["PATH"] = str(Path(FFMPEG).parent) + os.pathsep + os.environ["PATH"]
+use_certifi()
+
+import whisperx  # noqa: E402
+
+TIME = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
+
+
+def read_srt(path: Path):
+    """Cues as {start, end, text}. Tolerates a missing end on the last cue (a cut-off file)."""
+    cues = []
+    for block in re.split(r"\n\s*\n", path.read_text(encoding="utf-8-sig").strip()):
+        lines = [l.strip() for l in block.splitlines() if l.strip()]
+        for i, line in enumerate(lines):
+            m = TIME.search(line)
+            if m:
+                h1, m1, s1, ms1, h2, m2, s2, ms2 = map(int, m.groups())
+                text = " ".join(lines[i + 1 :]).strip()
+                if text:
+                    cues.append({"start": h1 * 3600 + m1 * 60 + s1 + ms1 / 1000, "end": h2 * 3600 + m2 * 60 + s2 + ms2 / 1000, "text": text})
+                break
+    return cues
+
+
+def main() -> None:
+    for stream in (_sys.stdout, _sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except (AttributeError, ValueError):
+            pass
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("media")
+    parser.add_argument("srt")
+    parser.add_argument("--language", default=None, help="e.g. ml, hi, ta, en. Default: the language chosen at setup")
+    parser.add_argument("--project", default=None)
+    parser.add_argument("--whole", action="store_true", help="ignore the cue times; align the whole script against the whole clip")
+    parser.add_argument("--pad", type=float, default=0.6, help="seconds of slack around each cue")
+    args = parser.parse_args()
+
+    media = Path(args.media)
+    out_dir = (Path("work") / args.project / media.stem) if args.project else (Path("work") / media.stem)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    started = time.time()
+
+    cues = read_srt(Path(args.srt))
+    if not cues:
+        raise SystemExit("no cues found in the .srt")
+    audio = whisperx.load_audio(str(media))
+    duration = len(audio) / 16000
+
+    # Pad each cue a little past its own times, even into a neighbour's window: hand-made .srt cues often
+    # start a few hundred ms after the speech does, and a window clipped to the cue pins those first
+    # words onto its edge (they show up with a score near 0 at exactly the cue start).
+    segments = []
+    if args.whole:
+        # One window over the whole clip: the cue times are not trusted at all, only the words and their order.
+        segments = [{"start": 0.0, "end": duration, "text": " ".join(c["text"] for c in cues)}]
+    for c in ([] if args.whole else cues):
+        lo = max(0.0, c["start"] - args.pad)
+        hi = min(duration, c["end"] + args.pad)
+        segments.append({"start": lo, "end": max(hi, lo + 0.2), "text": c["text"]})
+
+    args.language = lang(args.language or studio_language() or "en")["code"]
+    align_model, metadata = load_aligner(whisperx, args.language)
+    if align_model is None:
+        raise SystemExit(f"No word-timing model for '{args.language}', so a script cannot be matched to the audio. Transcribe instead: python scripts/transcribe.py <media>")
+    aligned = whisperx.align(segments, align_model, metadata, audio, device="cpu", return_char_alignments=False)
+
+    words = []
+    for segment in aligned["segments"]:
+        for w in segment.get("words", []):
+            words.append(
+                {
+                    "word": w["word"],
+                    "start": round(w["start"], 3) if "start" in w else None,
+                    "end": round(w["end"], 3) if "end" in w else None,
+                    "score": round(w["score"], 3) if "score" in w else None,
+                }
+            )
+
+    output = {
+        "source": str(media),
+        "language": args.language,
+        "model": f"forced-alignment:{metadata.get('language', args.language)} from {Path(args.srt).name}",
+        "durationSec": round(duration, 3),
+        "scriptCoversSec": round(cues[-1]["end"], 3),
+        "text": " ".join(c["text"] for c in cues),
+        "words": words,
+    }
+    out_file = out_dir / "words.json"
+    out_file.write_text(json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    unplaced = [w["word"] for w in words if w["start"] is None]
+    weak = [f'{w["word"]}@{w["start"]}' for w in words if w["score"] is not None and w["score"] < 0.15]
+    if weak:
+        print(f"low-confidence words (check by ear): {', '.join(weak)}")
+    print(f"cues: {len(cues)} | words: {len(words)} | unplaced: {len(unplaced)} {unplaced if unplaced else ''}")
+    if cues[-1]["end"] < duration - 1:
+        print(f"note: the script ends at {cues[-1]['end']:.1f}s but the video runs to {duration:.1f}s")
+    print(f"wrote {out_file} in {time.time() - started:.0f}s")
+
+
+if __name__ == "__main__":
+    main()

package/template/scripts/doctor.mjs

@@ -19,7 +19,7 @@
  * something core is broken. Optional pieces report but never fail the run.
  */
 import {execFileSync} from 'node:child_process';
-import {existsSync, readdirSync} from 'node:fs';
+import {existsSync, readdirSync, readFileSync} from 'node:fs';
 import {join, dirname, resolve} from 'node:path';
 import {fileURLToPath} from 'node:url';
 
@@ -85,6 +85,32 @@ if (venvOk) {
   }
 }
 
+/* ------------------------------------------- language models and motion tracking (setup) */
+if (venvOk) {
+  // The language chosen at setup decides which models must already be here (scripts/setup_models.py).
+  // Missing ones are not an error until a video needs them — but a big download mid-edit is the thing setup
+  // exists to prevent, so say so now.
+  let language = null;
+  try {
+    language = JSON.parse(readFileSync(join(ROOT, 'studio.config.json'), 'utf8')).language;
+  } catch {}
+  record('languages', 'Video language', !!language, language ? language : 'not chosen yet',
+    '.venv/Scripts/python scripts/setup_models.py --language <code>   (en hi hinglish te ml ta kn mr bn …)', false);
+  const hub = join(ROOT, '.models', 'huggingface', 'hub');
+  const cached = existsSync(hub) ? readdirSync(hub) : [];
+  if (language) {
+    const want = language === 'en' ? 'small' : 'large-v3';
+    const has = cached.some((d) => d.includes(`faster-whisper-${want}`));
+    record('languages', `Transcriber (${want})`, has, has ? 'downloaded' : 'not downloaded yet',
+      `.venv/Scripts/python scripts/setup_models.py --language ${language}`, false);
+  }
+  const cv = tryRun(VENV, ['-c', 'import cv2'], () => 'ok');
+  const seg = cached.some((d) => d.includes('BiRefNet'));
+  record('tracking', 'Motion tracking', cv.ok && seg,
+    cv.ok ? (seg ? 'ready' : 'person-finding model not downloaded') : 'OpenCV not installed',
+    cv.ok ? '.venv/Scripts/python scripts/setup_models.py --language <code>' : '.venv/Scripts/pip install -r requirements.txt', false);
+}
+
 /* ------------------------------------------------- the python half, script by script */
 if (venvOk) {
   // Importing whisperx is not the same as the scripts running. Compile each one, which

package/template/scripts/lib/languages.py

@@ -0,0 +1,68 @@
+"""The languages the studio is set up for, and what each one needs.
+
+One table, read by setup (scripts/setup_models.py), the transcriber and the script aligner, so they can
+never disagree about which model a language uses.
+
+  whisper  the transcription model. "small" is fine for English; Indian languages need "large-v3" (on a
+           Malayalam take the small model ignored the speech and invented English).
+  align    the word-timing model: None = WhisperX's built-in one for that language; a name = a public
+           wav2vec2 model for languages WhisperX does not cover; False = none exists, use the transcriber's
+           own (looser) timings.
+  tested   True where that combination has been run end to end on real footage. Untested ones are offered
+           as "word timing: experimental" and fall back to the transcriber's timings if they fail.
+
+The person's choice is saved in studio.config.json at the studio root:  {"language": "ml"}.
+"""
+
+import json
+from pathlib import Path
+
+LANGS = {
+    "en": {"name": "English", "whisper": "small", "align": None, "tested": True},
+    "hi": {"name": "Hindi", "whisper": "large-v3", "align": None, "tested": False},
+    # Hinglish is transcribed as Hindi (Whisper writes it in Devanagari) — for captions in English letters,
+    # share the script: aligning a known script is how code-mixed speech comes out right.
+    "hinglish": {"name": "Hinglish", "whisper": "large-v3", "align": None, "code": "hi", "tested": False},
+    "te": {"name": "Telugu", "whisper": "large-v3", "align": None, "tested": False},
+    "ml": {"name": "Malayalam", "whisper": "large-v3", "align": None, "tested": True},
+    "ur": {"name": "Urdu", "whisper": "large-v3", "align": None, "tested": False},
+    "ta": {"name": "Tamil", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-tamil-tam-250", "tested": False},
+    "kn": {"name": "Kannada", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-kannada-knm-560", "tested": False},
+    "mr": {"name": "Marathi", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-marathi-mrm-100", "tested": False},
+    "bn": {"name": "Bengali", "whisper": "large-v3", "align": "arijitx/wav2vec2-xls-r-300m-bengali", "tested": False},
+    "gu": {"name": "Gujarati", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-gujarati-gnm-100", "tested": False},
+    "pa": {"name": "Punjabi", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-punjabi-pam-10", "tested": False},
+}
+
+CONFIG = "studio.config.json"
+
+
+def lang(code: str | None) -> dict:
+    """What a language needs. Any other language: the large model, WhisperX's aligner if it has one."""
+    code = (code or "en").strip().lower()
+    info = dict(LANGS.get(code) or {"name": code, "whisper": "large-v3", "align": None, "tested": False})
+    info.setdefault("code", code)
+    return info
+
+
+def studio_language(root: Path | None = None) -> str | None:
+    """The language chosen at setup, if any."""
+    path = (root or Path.cwd()) / CONFIG
+    try:
+        return json.loads(path.read_text(encoding="utf-8")).get("language")
+    except (OSError, ValueError):
+        return None
+
+
+def load_aligner(whisperx, code: str):
+    """The word-timing model for a language, or (None, None) when there is none / it cannot load."""
+    info = lang(code)
+    if info["align"] is False:
+        return None, None
+    try:
+        if info["align"]:
+            return whisperx.load_align_model(language_code=info["code"], device="cpu", model_name=info["align"])
+        return whisperx.load_align_model(language_code=info["code"], device="cpu")
+    except Exception as exc:  # noqa: BLE001 — any failure here means "no aligner", not a crash
+        print(f"note: no word-timing model for {info['name']} ({exc.__class__.__name__}); using the transcriber's own timings.")
+        return None, None