creator-editing-studio 1.3.5 → 1.3.7

package/template/scripts/doctor.mjs

@@ -1,141 +1,167 @@
-#!/usr/bin/env node
-/**
- * Does this machine actually have everything the studio claims to do?
- *
- *   npm run doctor
- *
- * This exists because a working setup went quiet. WhisperX had been installed and used
- * (there are words.json files from a fortnight earlier), then the Python environment
- * underneath it was emptied, and nothing noticed until a job needed a transcript. The
- * code was fine the whole time. The environment was not, and there was no way to tell
- * except by running something and watching it fail.
- *
- * So this checks CAPABILITIES rather than files: it runs the binaries, imports the
- * modules, and reports what you can and cannot do right now. A capability that is
- * missing is reported with the command that fixes it, because "not installed" with no
- * next step is what turns a five-minute problem into a phone call.
- *
- * Exit code is 0 when everything the studio needs for a render is present, 1 when
- * something core is broken. Optional pieces report but never fail the run.
- */
-import {execFileSync} from 'node:child_process';
-import {existsSync, readdirSync} from 'node:fs';
-import {join, dirname, resolve} from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-// fileURLToPath, not URL.pathname: a path with a space in it comes back percent-encoded
-// from pathname, so every existsSync below silently answers false and the report claims
-// a working studio is broken.
-const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
-const VENV = process.platform === 'win32'
-  ? join(ROOT, '.venv', 'Scripts', 'python.exe')
-  : join(ROOT, '.venv', 'bin', 'python');
-
-const results = [];
-const record = (area, name, ok, detail, fix, core = true) =>
-  results.push({area, name, ok, detail, fix, core});
-
-const tryRun = (bin, args, pick = (s) => s.trim().split('\n')[0]) => {
-  try {
-    return {ok: true, out: pick(execFileSync(bin, args, {encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']}))};
-  } catch (e) {
-    return {ok: false, out: (e.stderr || e.message || '').toString().trim().split('\n')[0]};
-  }
-};
-
-/* ------------------------------------------------------------------ the engine */
-const node = process.versions.node;
-const nodeMajor = Number(node.split('.')[0]);
-record('engine', 'Node', nodeMajor >= 20, `v${node}`, 'Install the LTS build from nodejs.org');
-
-const hasModules = existsSync(join(ROOT, 'node_modules', 'remotion'));
-const pkgCount = existsSync(join(ROOT, 'node_modules')) ? readdirSync(join(ROOT, 'node_modules')).length : 0;
-record('engine', 'Video engine', hasModules, hasModules ? `${pkgCount} packages` : 'node_modules/remotion missing', 'npm install');
-
-/* ------------------------------------------------------------- media handling */
-let FFMPEG = 'ffmpeg', FFPROBE = 'ffprobe';
-try {
-  const media = await import('./lib/media.mjs');
-  FFMPEG = media.FFMPEG;
-  FFPROBE = media.FFPROBE;
-} catch { /* fall back to PATH and let the run below report it */ }
-
-for (const [name, bin] of [['ffmpeg', FFMPEG], ['ffprobe', FFPROBE]]) {
-  const r = tryRun(bin, ['-version']);
-  record('media', name, r.ok, r.ok ? r.out.split(' ').slice(0, 3).join(' ') : 'not runnable',
-    'Set $FFMPEG / $FFPROBE, or install ffmpeg so it is on PATH');
-}
-
-/* ------------------------------------------------------ the transcriber (optional) */
-const venvOk = existsSync(VENV);
-record('transcriber', 'Python 3.11/3.12 venv', venvOk, venvOk ? VENV.replace(ROOT, '.') : '.venv missing',
-  'py -3.12 -m venv .venv   (3.13+ will not build WhisperX)', false);
-
-if (venvOk) {
-  const ver = tryRun(VENV, ['-c', 'import sys;print(sys.version.split()[0])']);
-  const minor = Number((ver.out || '0.0').split('.')[1]);
-  record('transcriber', 'Python version', ver.ok && (minor === 11 || minor === 12),
-    ver.out, 'Rebuild the venv with python 3.11 or 3.12', false);
-
-  // Every module the Python half imports, not just the headline one.
-  for (const mod of ['whisperx', 'torch', 'numpy', 'PIL', 'certifi']) {
-    const r = tryRun(VENV, ['-c', `import ${mod}`], () => 'ok');
-    record('transcriber', mod, r.ok, r.ok ? 'ok' : 'not importable',
-      '.venv/Scripts/pip install -r requirements.txt', false);
-  }
-}
-
-/* ------------------------------------------------- the python half, script by script */
-if (venvOk) {
-  // Importing whisperx is not the same as the scripts running. Compile each one, which
-  // catches a missing dependency or a syntax error without doing any of the slow work.
-  const py = readdirSync(join(ROOT, 'scripts')).filter((f) => f.endsWith('.py'));
-  const bad = [];
-  for (const f of py) {
-    const r = tryRun(VENV, ['-m', 'py_compile', join(ROOT, 'scripts', f)], () => 'ok');
-    if (!r.ok) bad.push(f);
-  }
-  record('transcriber', 'Python scripts', bad.length === 0,
-    bad.length ? `${bad.join(', ')} will not compile` : `${py.length} compile clean`,
-    '.venv/Scripts/pip install -r requirements.txt', false);
-}
-
-/* ----------------------------------------------------------------------- brand */
-const tokens = existsSync(join(ROOT, 'src', 'design', 'tokens.ts'));
-record('brand', 'Design tokens', tokens, tokens ? 'src/design/tokens.ts' : 'missing',
-  'Restore src/design/tokens.ts, which is where the whole look lives');
-
-// A missing font file does not fail a render: it silently swaps in a fallback, and the
-// video comes out looking almost right, which is worse than an error.
-const fontDir = join(ROOT, 'assets', 'fonts');
-const fonts = existsSync(fontDir) ? readdirSync(fontDir).filter((f) => /\.(ttf|otf|woff2?)$/i.test(f)) : [];
-record('brand', 'Fonts', fonts.length > 0,
-  fonts.length ? `${fonts.length} file(s): ${fonts.slice(0, 2).join(', ')}` : 'assets/fonts is empty',
-  'Put the brand font files back in assets/fonts, or a render falls back to a system face');
-
-/* --------------------------------------------------------------------- report */
-const W = 22;
-let lastArea = '';
-for (const r of results) {
-  if (r.area !== lastArea) { console.log(`\n${r.area.toUpperCase()}`); lastArea = r.area; }
-  const mark = r.ok ? '  ok  ' : (r.core ? ' FAIL ' : ' --   ');
-  console.log(`${mark} ${r.name.padEnd(W)} ${r.detail}`);
-  if (!r.ok) console.log(`       ${''.padEnd(W)} fix: ${r.fix}`);
-}
-
-const brokenCore = results.filter((r) => !r.ok && r.core);
-const brokenOpt = results.filter((r) => !r.ok && !r.core);
-
-console.log('\n' + '-'.repeat(64));
-if (!brokenCore.length && !brokenOpt.length) {
-  console.log('Everything is present. You can cut, render and caption.');
-} else if (!brokenCore.length) {
-  console.log(`Rendering works. ${brokenOpt.length} optional piece(s) missing:`);
-  console.log(`  ${[...new Set(brokenOpt.map((r) => r.fix))].join('\n  ')}`);
-  console.log('Without these: no word-level captions, no auto cut detection, no green-screen cutouts.');
-} else {
-  console.log(`${brokenCore.length} thing(s) needed for a render are missing. Fix these first:`);
-  console.log(`  ${[...new Set(brokenCore.map((r) => r.fix))].join('\n  ')}`);
-}
-
-process.exit(brokenCore.length ? 1 : 0);
+#!/usr/bin/env node
+/**
+ * Does this machine actually have everything the studio claims to do?
+ *
+ *   npm run doctor
+ *
+ * This exists because a working setup went quiet. WhisperX had been installed and used
+ * (there are words.json files from a fortnight earlier), then the Python environment
+ * underneath it was emptied, and nothing noticed until a job needed a transcript. The
+ * code was fine the whole time. The environment was not, and there was no way to tell
+ * except by running something and watching it fail.
+ *
+ * So this checks CAPABILITIES rather than files: it runs the binaries, imports the
+ * modules, and reports what you can and cannot do right now. A capability that is
+ * missing is reported with the command that fixes it, because "not installed" with no
+ * next step is what turns a five-minute problem into a phone call.
+ *
+ * Exit code is 0 when everything the studio needs for a render is present, 1 when
+ * something core is broken. Optional pieces report but never fail the run.
+ */
+import {execFileSync} from 'node:child_process';
+import {existsSync, readdirSync, readFileSync} from 'node:fs';
+import {join, dirname, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+// fileURLToPath, not URL.pathname: a path with a space in it comes back percent-encoded
+// from pathname, so every existsSync below silently answers false and the report claims
+// a working studio is broken.
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const VENV = process.platform === 'win32'
+  ? join(ROOT, '.venv', 'Scripts', 'python.exe')
+  : join(ROOT, '.venv', 'bin', 'python');
+
+const results = [];
+const record = (area, name, ok, detail, fix, core = true) =>
+  results.push({area, name, ok, detail, fix, core});
+
+const tryRun = (bin, args, pick = (s) => s.trim().split('\n')[0]) => {
+  try {
+    return {ok: true, out: pick(execFileSync(bin, args, {encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']}))};
+  } catch (e) {
+    return {ok: false, out: (e.stderr || e.message || '').toString().trim().split('\n')[0]};
+  }
+};
+
+/* ------------------------------------------------------------------ the engine */
+const node = process.versions.node;
+const nodeMajor = Number(node.split('.')[0]);
+record('engine', 'Node', nodeMajor >= 20, `v${node}`, 'Install the LTS build from nodejs.org');
+
+const hasModules = existsSync(join(ROOT, 'node_modules', 'remotion'));
+const pkgCount = existsSync(join(ROOT, 'node_modules')) ? readdirSync(join(ROOT, 'node_modules')).length : 0;
+record('engine', 'Video engine', hasModules, hasModules ? `${pkgCount} packages` : 'node_modules/remotion missing', 'npm install');
+
+/* ------------------------------------------------------------- media handling */
+let FFMPEG = 'ffmpeg', FFPROBE = 'ffprobe';
+try {
+  const media = await import('./lib/media.mjs');
+  FFMPEG = media.FFMPEG;
+  FFPROBE = media.FFPROBE;
+} catch { /* fall back to PATH and let the run below report it */ }
+
+for (const [name, bin] of [['ffmpeg', FFMPEG], ['ffprobe', FFPROBE]]) {
+  const r = tryRun(bin, ['-version']);
+  record('media', name, r.ok, r.ok ? r.out.split(' ').slice(0, 3).join(' ') : 'not runnable',
+    'Set $FFMPEG / $FFPROBE, or install ffmpeg so it is on PATH');
+}
+
+/* ------------------------------------------------------ the transcriber (optional) */
+const venvOk = existsSync(VENV);
+record('transcriber', 'Python 3.11/3.12 venv', venvOk, venvOk ? VENV.replace(ROOT, '.') : '.venv missing',
+  'py -3.12 -m venv .venv   (3.13+ will not build WhisperX)', false);
+
+if (venvOk) {
+  const ver = tryRun(VENV, ['-c', 'import sys;print(sys.version.split()[0])']);
+  const minor = Number((ver.out || '0.0').split('.')[1]);
+  record('transcriber', 'Python version', ver.ok && (minor === 11 || minor === 12),
+    ver.out, 'Rebuild the venv with python 3.11 or 3.12', false);
+
+  // Every module the Python half imports, not just the headline one.
+  for (const mod of ['whisperx', 'torch', 'numpy', 'PIL', 'certifi']) {
+    const r = tryRun(VENV, ['-c', `import ${mod}`], () => 'ok');
+    record('transcriber', mod, r.ok, r.ok ? 'ok' : 'not importable',
+      '.venv/Scripts/pip install -r requirements.txt', false);
+  }
+}
+
+/* ------------------------------------------- language models and motion tracking (setup) */
+if (venvOk) {
+  // The language chosen at setup decides which models must already be here (scripts/setup_models.py).
+  // Missing ones are not an error until a video needs them — but a big download mid-edit is the thing setup
+  // exists to prevent, so say so now.
+  let language = null;
+  try {
+    language = JSON.parse(readFileSync(join(ROOT, 'studio.config.json'), 'utf8')).language;
+  } catch {}
+  record('languages', 'Video language', !!language, language ? language : 'not chosen yet',
+    '.venv/Scripts/python scripts/setup_models.py --language <code>   (en hi hinglish te ml ta kn mr bn …)', false);
+  const hub = join(ROOT, '.models', 'huggingface', 'hub');
+  const cached = existsSync(hub) ? readdirSync(hub) : [];
+  if (language) {
+    const want = language === 'en' ? 'small' : 'large-v3';
+    const has = cached.some((d) => d.includes(`faster-whisper-${want}`));
+    record('languages', `Transcriber (${want})`, has, has ? 'downloaded' : 'not downloaded yet',
+      `.venv/Scripts/python scripts/setup_models.py --language ${language}`, false);
+  }
+  const cv = tryRun(VENV, ['-c', 'import cv2'], () => 'ok');
+  const seg = cached.some((d) => d.includes('BiRefNet'));
+  record('tracking', 'Motion tracking', cv.ok && seg,
+    cv.ok ? (seg ? 'ready' : 'person-finding model not downloaded') : 'OpenCV not installed',
+    cv.ok ? '.venv/Scripts/python scripts/setup_models.py --language <code>' : '.venv/Scripts/pip install -r requirements.txt', false);
+}
+
+/* ------------------------------------------------- the python half, script by script */
+if (venvOk) {
+  // Importing whisperx is not the same as the scripts running. Compile each one, which
+  // catches a missing dependency or a syntax error without doing any of the slow work.
+  const py = readdirSync(join(ROOT, 'scripts')).filter((f) => f.endsWith('.py'));
+  const bad = [];
+  for (const f of py) {
+    const r = tryRun(VENV, ['-m', 'py_compile', join(ROOT, 'scripts', f)], () => 'ok');
+    if (!r.ok) bad.push(f);
+  }
+  record('transcriber', 'Python scripts', bad.length === 0,
+    bad.length ? `${bad.join(', ')} will not compile` : `${py.length} compile clean`,
+    '.venv/Scripts/pip install -r requirements.txt', false);
+}
+
+/* ----------------------------------------------------------------------- brand */
+const tokens = existsSync(join(ROOT, 'src', 'design', 'tokens.ts'));
+record('brand', 'Design tokens', tokens, tokens ? 'src/design/tokens.ts' : 'missing',
+  'Restore src/design/tokens.ts, which is where the whole look lives');
+
+// A missing font file does not fail a render: it silently swaps in a fallback, and the
+// video comes out looking almost right, which is worse than an error.
+const fontDir = join(ROOT, 'assets', 'fonts');
+const fonts = existsSync(fontDir) ? readdirSync(fontDir).filter((f) => /\.(ttf|otf|woff2?)$/i.test(f)) : [];
+record('brand', 'Fonts', fonts.length > 0,
+  fonts.length ? `${fonts.length} file(s): ${fonts.slice(0, 2).join(', ')}` : 'assets/fonts is empty',
+  'Put the brand font files back in assets/fonts, or a render falls back to a system face');
+
+/* --------------------------------------------------------------------- report */
+const W = 22;
+let lastArea = '';
+for (const r of results) {
+  if (r.area !== lastArea) { console.log(`\n${r.area.toUpperCase()}`); lastArea = r.area; }
+  const mark = r.ok ? '  ok  ' : (r.core ? ' FAIL ' : ' --   ');
+  console.log(`${mark} ${r.name.padEnd(W)} ${r.detail}`);
+  if (!r.ok) console.log(`       ${''.padEnd(W)} fix: ${r.fix}`);
+}
+
+const brokenCore = results.filter((r) => !r.ok && r.core);
+const brokenOpt = results.filter((r) => !r.ok && !r.core);
+
+console.log('\n' + '-'.repeat(64));
+if (!brokenCore.length && !brokenOpt.length) {
+  console.log('Everything is present. You can cut, render and caption.');
+} else if (!brokenCore.length) {
+  console.log(`Rendering works. ${brokenOpt.length} optional piece(s) missing:`);
+  console.log(`  ${[...new Set(brokenOpt.map((r) => r.fix))].join('\n  ')}`);
+  console.log('Without these: no word-level captions, no auto cut detection, no green-screen cutouts.');
+} else {
+  console.log(`${brokenCore.length} thing(s) needed for a render are missing. Fix these first:`);
+  console.log(`  ${[...new Set(brokenCore.map((r) => r.fix))].join('\n  ')}`);
+}
+
+process.exit(brokenCore.length ? 1 : 0);

package/template/scripts/find_cuts.py

@@ -1,45 +1,45 @@
-"""Find jump cuts in a talking-head clip (frames where the picture jumps instead of moving).
-
-Usage (from the project root):
-    python scripts/find_cuts.py <video> <out.json>
-
-Writes a JSON array of frame numbers: each is the first frame AFTER a cut. Used to
- - punch in/out on each cut (the Stage `punchIns` prop), so cuts read as intentional camera changes
- - tell motion-check these changes are in the footage (--allow-file)
-"""
-
-import json
-import subprocess
-import sys
-
-import numpy as np
-import sys as _sys, pathlib as _pl
-_sys.path.insert(0, str(_pl.Path(__file__).resolve().parent / 'lib'))
-from media import FFMPEG, FFPROBE  # resolved per platform - never hardcode a tool path
-
-
-
-def main() -> None:
-    video, out_path = sys.argv[1], sys.argv[2]
-    raw = subprocess.run(
-        [FFMPEG, "-v", "error", "-i", video, "-vf", "scale=320:180", "-f", "rawvideo", "-pix_fmt", "gray", "-"],
-        capture_output=True,
-        check=True,
-    ).stdout
-    frames = np.frombuffer(raw, np.uint8).reshape(-1, 180, 320).astype(np.float32)
-    change = np.abs(np.diff(frames, axis=0)).mean(axis=(1, 2))
-    typical = float(np.median(change))
-
-    cuts = []
-    for i in range(2, len(change) - 2):
-        neighbours = float(np.median(np.concatenate([change[i - 2 : i], change[i + 1 : i + 3]])))
-        # A cut: one transition far larger than both the clip's typical motion and its neighbours.
-        if change[i] > 6 and change[i] > 4 * max(neighbours, typical):
-            cuts.append(i + 1)
-
-    json.dump(cuts, open(out_path, "w"), indent=0)
-    print(f"{len(cuts)} cuts -> {out_path}: {cuts}")
-
-
-if __name__ == "__main__":
-    main()
+"""Find jump cuts in a talking-head clip (frames where the picture jumps instead of moving).
+
+Usage (from the project root):
+    python scripts/find_cuts.py <video> <out.json>
+
+Writes a JSON array of frame numbers: each is the first frame AFTER a cut. Used to
+ - punch in/out on each cut (the Stage `punchIns` prop), so cuts read as intentional camera changes
+ - tell motion-check these changes are in the footage (--allow-file)
+"""
+
+import json
+import subprocess
+import sys
+
+import numpy as np
+import sys as _sys, pathlib as _pl
+_sys.path.insert(0, str(_pl.Path(__file__).resolve().parent / 'lib'))
+from media import FFMPEG, FFPROBE  # resolved per platform - never hardcode a tool path
+
+
+
+def main() -> None:
+    video, out_path = sys.argv[1], sys.argv[2]
+    raw = subprocess.run(
+        [FFMPEG, "-v", "error", "-i", video, "-vf", "scale=320:180", "-f", "rawvideo", "-pix_fmt", "gray", "-"],
+        capture_output=True,
+        check=True,
+    ).stdout
+    frames = np.frombuffer(raw, np.uint8).reshape(-1, 180, 320).astype(np.float32)
+    change = np.abs(np.diff(frames, axis=0)).mean(axis=(1, 2))
+    typical = float(np.median(change))
+
+    cuts = []
+    for i in range(2, len(change) - 2):
+        neighbours = float(np.median(np.concatenate([change[i - 2 : i], change[i + 1 : i + 3]])))
+        # A cut: one transition far larger than both the clip's typical motion and its neighbours.
+        if change[i] > 6 and change[i] > 4 * max(neighbours, typical):
+            cuts.append(i + 1)
+
+    json.dump(cuts, open(out_path, "w"), indent=0)
+    print(f"{len(cuts)} cuts -> {out_path}: {cuts}")
+
+
+if __name__ == "__main__":
+    main()

package/template/scripts/lib/languages.py

@@ -0,0 +1,68 @@
+"""The languages the studio is set up for, and what each one needs.
+
+One table, read by setup (scripts/setup_models.py), the transcriber and the script aligner, so they can
+never disagree about which model a language uses.
+
+  whisper  the transcription model. "small" is fine for English; Indian languages need "large-v3" (on a
+           Malayalam take the small model ignored the speech and invented English).
+  align    the word-timing model: None = WhisperX's built-in one for that language; a name = a public
+           wav2vec2 model for languages WhisperX does not cover; False = none exists, use the transcriber's
+           own (looser) timings.
+  tested   True where that combination has been run end to end on real footage. Untested ones are offered
+           as "word timing: experimental" and fall back to the transcriber's timings if they fail.
+
+The person's choice is saved in studio.config.json at the studio root:  {"language": "ml"}.
+"""
+
+import json
+from pathlib import Path
+
+LANGS = {
+    "en": {"name": "English", "whisper": "small", "align": None, "tested": True},
+    "hi": {"name": "Hindi", "whisper": "large-v3", "align": None, "tested": False},
+    # Hinglish is transcribed as Hindi (Whisper writes it in Devanagari) — for captions in English letters,
+    # share the script: aligning a known script is how code-mixed speech comes out right.
+    "hinglish": {"name": "Hinglish", "whisper": "large-v3", "align": None, "code": "hi", "tested": False},
+    "te": {"name": "Telugu", "whisper": "large-v3", "align": None, "tested": False},
+    "ml": {"name": "Malayalam", "whisper": "large-v3", "align": None, "tested": True},
+    "ur": {"name": "Urdu", "whisper": "large-v3", "align": None, "tested": False},
+    "ta": {"name": "Tamil", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-tamil-tam-250", "tested": False},
+    "kn": {"name": "Kannada", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-kannada-knm-560", "tested": False},
+    "mr": {"name": "Marathi", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-marathi-mrm-100", "tested": False},
+    "bn": {"name": "Bengali", "whisper": "large-v3", "align": "arijitx/wav2vec2-xls-r-300m-bengali", "tested": False},
+    "gu": {"name": "Gujarati", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-gujarati-gnm-100", "tested": False},
+    "pa": {"name": "Punjabi", "whisper": "large-v3", "align": "Harveenchadha/vakyansh-wav2vec2-punjabi-pam-10", "tested": False},
+}
+
+CONFIG = "studio.config.json"
+
+
+def lang(code: str | None) -> dict:
+    """What a language needs. Any other language: the large model, WhisperX's aligner if it has one."""
+    code = (code or "en").strip().lower()
+    info = dict(LANGS.get(code) or {"name": code, "whisper": "large-v3", "align": None, "tested": False})
+    info.setdefault("code", code)
+    return info
+
+
+def studio_language(root: Path | None = None) -> str | None:
+    """The language chosen at setup, if any."""
+    path = (root or Path.cwd()) / CONFIG
+    try:
+        return json.loads(path.read_text(encoding="utf-8")).get("language")
+    except (OSError, ValueError):
+        return None
+
+
+def load_aligner(whisperx, code: str):
+    """The word-timing model for a language, or (None, None) when there is none / it cannot load."""
+    info = lang(code)
+    if info["align"] is False:
+        return None, None
+    try:
+        if info["align"]:
+            return whisperx.load_align_model(language_code=info["code"], device="cpu", model_name=info["align"])
+        return whisperx.load_align_model(language_code=info["code"], device="cpu")
+    except Exception as exc:  # noqa: BLE001 — any failure here means "no aligner", not a crash
+        print(f"note: no word-timing model for {info['name']} ({exc.__class__.__name__}); using the transcriber's own timings.")
+        return None, None