creator-editing-studio 1.3.6 → 1.3.7

package/template/scripts/transcribe.py

@@ -17,6 +17,7 @@ from pathlib import Path
 import sys as _sys  # noqa: E402
 _sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
 from media import FFMPEG, use_certifi  # noqa: E402
+from languages import lang, load_aligner, studio_language  # noqa: E402
 
 # Models are large; cache them inside the studio so they survive and are easy to find.
 _MODELS = Path(__file__).resolve().parent.parent / ".models"
@@ -49,12 +50,46 @@ except ModuleNotFoundError as exc:  # pragma: no cover
     raise SystemExit(2)
 
 
+# Indian languages: the small model cannot do them. On Malayalam it ignored the speech, invented one
+# English sentence and repeated it, then drifted into Devanagari (2026-10-17). large-v3 is the
+# smallest model that transcribes these reliably, so it is the default for them.
+INDIC = {"ml", "ta", "te", "kn", "hi", "bn", "mr", "gu", "pa", "or", "as", "ur", "ne", "si"}
+
+# The script each language must come out in. A transcript mostly in another script is a model
+# failure (translated, or a neighbouring language), never a usable result, so it stops here.
+SCRIPTS = {
+    "ml": (0x0D00, 0x0D7F), "ta": (0x0B80, 0x0BFF), "te": (0x0C00, 0x0C7F), "kn": (0x0C80, 0x0CFF),
+    "hi": (0x0900, 0x097F), "mr": (0x0900, 0x097F), "ne": (0x0900, 0x097F), "bn": (0x0980, 0x09FF),
+    "as": (0x0980, 0x09FF), "gu": (0x0A80, 0x0AFF), "pa": (0x0A00, 0x0A7F), "or": (0x0B00, 0x0B7F),
+    "si": (0x0D80, 0x0DFF), "ur": (0x0600, 0x06FF),
+}
+
+
+def script_share(text: str, language: str) -> float | None:
+    """Share of letters in the language's own script (Latin words like brand names are allowed)."""
+    span = SCRIPTS.get(language)
+    if not span:
+        return None
+    letters = [c for c in text if c.isalpha()]
+    if not letters:
+        return 0.0
+    return sum(span[0] <= ord(c) <= span[1] for c in letters) / len(letters)
+
+
 def main() -> None:
+    # Windows consoles default to cp1252, which cannot print Malayalam: the script used to finish
+    # the transcription and then crash on its own summary line.
+    for stream in (_sys.stdout, _sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")
+        except (AttributeError, ValueError):
+            pass
+
     parser = argparse.ArgumentParser()
     parser.add_argument("media")
-    parser.add_argument("--language", default=None, help="e.g. en, hi. Detected when omitted.")
+    parser.add_argument("--language", default=None, help="e.g. en, hi, ml. Default: the language chosen at setup (studio.config.json), else detected.")
     parser.add_argument("--project", default=None, help="Video project name: writes to work/<project>/<clip>/")
-    parser.add_argument("--model", default="small", help="Whisper size: small (fast on CPU) or medium (more accurate)")
+    parser.add_argument("--model", default=None, help="Whisper size. Default: what the language needs (scripts/lib/languages.py)")
     args = parser.parse_args()
 
     media = Path(args.media)
@@ -62,13 +97,35 @@ def main() -> None:
     out_dir.mkdir(parents=True, exist_ok=True)
     started = time.time()
 
+    if args.language is None:
+        args.language = studio_language()
+    info = lang(args.language) if args.language else None
+    if info:
+        args.language = info["code"]  # Hinglish is transcribed as Hindi
+    if args.model is None:
+        args.model = info["whisper"] if info else "small"
     audio = whisperx.load_audio(str(media))
     model = whisperx.load_model(args.model, device="cpu", compute_type="int8", language=args.language)
-    result = model.transcribe(audio, batch_size=4, language=args.language)
+    # task="transcribe" always: without it a weak model may answer in English instead.
+    result = model.transcribe(audio, batch_size=4, language=args.language, task="transcribe")
     language = result["language"]
 
-    align_model, metadata = whisperx.load_align_model(language_code=language, device="cpu")
-    aligned = whisperx.align(result["segments"], align_model, metadata, audio, device="cpu", return_char_alignments=False)
+    align_model, metadata = load_aligner(whisperx, language)
+    if align_model is not None:
+        aligned = whisperx.align(result["segments"], align_model, metadata, audio, device="cpu", return_char_alignments=False)
+    else:
+        # No word-timing model for this language: spread each segment's words across its time, by length.
+        # Looser than real alignment — good enough to cut on, worth checking where captions must land exactly.
+        aligned = {"segments": []}
+        for seg in result["segments"]:
+            ws = seg["text"].split()
+            total = sum(len(w) for w in ws) or 1
+            t, words_ = seg["start"], []
+            for w in ws:
+                d = (seg["end"] - seg["start"]) * len(w) / total
+                words_.append({"word": w, "start": t, "end": t + d})
+                t += d
+            aligned["segments"].append({**seg, "words": words_})
 
     words = []
     for segment in aligned["segments"]:
@@ -93,6 +150,18 @@ def main() -> None:
     out_file = out_dir / "words.json"
     out_file.write_text(json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8")
 
+    share = script_share(output["text"], language)
+    # Low bar on purpose: everyday Malayalam mixes in English words ("Instagram", "business"), but a
+    # failed run scores near zero (the small model's Malayalam came out at 0%).
+    if share is not None and share < 0.3:
+        print(
+            f"WARNING: only {share:.0%} of this transcript is in the {language} script, so the model did not "
+            f"really transcribe it (it translated, or heard another language). Do not cut from it. "
+            f"Re-run with a larger --model, and check --language is right.",
+            file=_sys.stderr,
+        )
+        raise SystemExit(3)
+
     unplaced = [w["word"] for w in words if w["start"] is None]
     print(f"language: {language} | words: {len(words)} | unplaced: {len(unplaced)} {unplaced if unplaced else ''}")
     print(f"text: {output['text']}")

package/template/src/components/HeadPin.tsx

@@ -0,0 +1,42 @@
+import React from 'react';
+import {useCurrentFrame} from 'remotion';
+import {EASE} from '../design/motion';
+import {progress} from '../lib/animate';
+import {pinStyle, sampleTrack, type TrackFile} from '../lib/track';
+
+/**
+ * Text or a graphic that rides with the speaker's head (motion tracking). Never a default — use it when
+ * someone asks for it, on a stretch that has been tracked with scripts/track_head.py.
+ *
+ *   <HeadPin track={headTrack} toSource={(f) => f + 72} from={5} to={53} dy={-330}>
+ *     <div style={…}>Big title</div>
+ *   </HeadPin>
+ *
+ * dx/dy place it relative to the head (at the head's starting size; they grow as the head comes closer).
+ * It follows two frames behind and takes half the head's tilt — attached, not glued. Outside the tracked
+ * frames it simply does not draw, so it can never float in the wrong place.
+ */
+export const HeadPin: React.FC<{
+  track: TrackFile;
+  /** The source frame of the tracked clip shown at a composition frame (null when the clip is not on screen). */
+  toSource: (frame: number) => number | null;
+  from: number;
+  to: number;
+  dx?: number;
+  dy?: number;
+  tilt?: number;
+  children: React.ReactNode;
+}> = ({track, toSource, from, to, dx = 0, dy = 0, tilt = 0.5, children}) => {
+  const f = useCurrentFrame();
+  if (f < from - 2 || f > to) return null;
+  const src = toSource(f);
+  const head = src === null ? null : sampleTrack(track, src, 2);
+  if (!head) return null;
+  const p = progress(f, from, 8, EASE.overshoot);
+  const out = 1 - progress(f, to - 5, 5, EASE.in);
+  return (
+    <div style={{...pinStyle(head, {dx, dy, tilt}), opacity: Math.min(1, p * 1.6) * out}}>
+      <div style={{transform: `scale(${0.85 + Math.min(1.05, p) * 0.15}) translateY(${(1 - Math.min(1, p)) * 20}px)`, display: 'flex', flexDirection: 'column', alignItems: 'center'}}>{children}</div>
+    </div>
+  );
+};

package/template/src/lib/track.ts

@@ -0,0 +1,67 @@
+/**
+ * Motion tracking: pin graphics to something moving in the footage — a head, from scripts/track_head.py.
+ *
+ *   const head = sampleTrack(track, sourceFrame, 2);          // null outside the tracked stretches
+ *   if (head) <div style={pinStyle(head, {dy: -330})}>…</div>
+ *
+ * A track holds SOURCE frames of one clip, in that clip's pixels; a composition showing the clip
+ * full-frame at the same size can use it directly. One file per video can hold several tracked stretches.
+ * `lag` follows a couple of frames behind, so a graphic feels attached by a soft spring, not glued on.
+ */
+import type {CSSProperties} from 'react';
+
+export type TrackPoint = {f: number; x: number; y: number; s: number; r: number};
+export type TrackFile = {fps: number; size: number[]; first: number; box: {w: number; h: number}; frames: TrackPoint[]};
+export type Pin = {x: number; y: number; s: number; r: number};
+
+/** The tracked point at a source frame, or null if that frame was not tracked. */
+function pointAt(t: TrackFile, f: number): TrackPoint | null {
+  const fr = Math.round(f);
+  let lo = 0;
+  let hi = t.frames.length - 1;
+  while (lo <= hi) {
+    const mid = (lo + hi) >> 1;
+    const m = t.frames[mid];
+    if (m.f === fr) return m;
+    if (m.f < fr) lo = mid + 1;
+    else hi = mid - 1;
+  }
+  return null;
+}
+
+/** Where the tracked thing is at a source frame, averaged over `lag` frames behind it; null if untracked. */
+export function sampleTrack(t: TrackFile, sourceFrame: number, lag = 0): Pin | null {
+  const pts = Array.from({length: lag + 1}, (_, k) => pointAt(t, sourceFrame - k)).filter((p): p is TrackPoint => !!p);
+  if (!pointAt(t, sourceFrame) || !pts.length) return null;
+  const avg = (k: 'x' | 'y' | 's' | 'r') => pts.reduce((sum, p) => sum + p[k], 0) / pts.length;
+  return {x: avg('x'), y: avg('y'), s: avg('s'), r: avg('r')};
+}
+
+/**
+ * Absolute position for a graphic pinned to the track. dx/dy are offsets at the starting size and grow with
+ * the head; `tilt` is how much of the head's tilt the graphic takes (a little reads as attached; all of it
+ * reads as stuck on).
+ */
+export function pinStyle(p: Pin, {dx = 0, dy = 0, tilt = 0.5}: {dx?: number; dy?: number; tilt?: number} = {}): CSSProperties {
+  return {
+    position: 'absolute',
+    left: p.x + dx * p.s,
+    top: p.y + dy * p.s,
+    transform: `translate(-50%, -50%) scale(${p.s}) rotate(${p.r * tilt}deg)`,
+    transformOrigin: '50% 50%',
+  };
+}
+
+/**
+ * How far the head has moved since a reference frame, as a CSS transform about the head — for "follow":
+ * a layout keeps its design but moves, scales and (partly) tilts with the head. Null if either is untracked.
+ */
+export function followTransform(t: TrackFile, sourceFrame: number, fromSourceFrame: number, tilt = 0.5): {transform: string; transformOrigin: string} | null {
+  const now = sampleTrack(t, sourceFrame, 2);
+  const at = sampleTrack(t, fromSourceFrame);
+  if (!now || !at) return null;
+  return {
+    transform: `translate(${now.x - at.x}px, ${now.y - at.y}px) scale(${now.s / at.s}) rotate(${(now.r - at.r) * tilt}deg)`,
+    transformOrigin: `${at.x}px ${at.y}px`,
+  };
+}

package/template/src/lib/tweaks.tsx

@@ -0,0 +1,98 @@
+/**
+ * Live-preview adjustments ("tweaks"): what the preview's instant controls change, applied to an edit
+ * without touching its code.
+ *
+ * An edit wraps itself in <TweaksProvider beatStarts={…} tweaks={…}> and wraps each layer that should
+ * respond in <Tweaked>. In the preview the tweaks arrive as props and change the video instantly; saved,
+ * they live in the video's tweaks.json and the final render reads them from there. No tweaks = the edit
+ * exactly as built.
+ *
+ * Rules that keep an edit correct whatever someone clicks:
+ *  - Anything that hides a join in the cut (a full-screen card) must NOT be inside a moving <Tweaked> — the
+ *    card's exact frames are what hide the join. Wrap only what sits ON it.
+ *  - Layers locked to the footage (a subject cutout) use <Tweaked move={false}> and read the tweak with
+ *    useHereTweak() to move only their graphics.
+ */
+import React, {createContext, useContext} from 'react';
+import {AbsoluteFill, Sequence, useCurrentFrame} from 'remotion';
+import {followTransform, type TrackFile} from './track';
+
+/**
+ * One beat's adjustments. All optional.
+ *   scale  size of text and graphics, 0.7–1.4          dy     up (−) / down (+), px, ±300
+ *   shift  frames later (+) / earlier (−), ±12          gain   that beat's sound effects, dB, −12…+6
+ *   mute   that beat's sound effects off                icon   swap a 3D icon: {cart: 'shop'}
+ *   follow the beat's graphics move, scale and partly tilt with the speaker's head (needs a head track)
+ */
+export type Tweak = {scale?: number; dy?: number; shift?: number; gain?: number; mute?: boolean; icon?: Record<string, string>; follow?: boolean};
+export type Tweaks = Record<string, Tweak>; // keyed by beat index
+
+export const TWEAK_LIMITS = {scale: [0.7, 1.4], dy: [-300, 300], shift: [-12, 12], gain: [-12, 6]} as const;
+
+/** track/toSource: optional head track for the "follow" tweak — the clip's source frame at a composition frame. */
+type Ctx = {tweaks: Tweaks; beatAt: (frame: number) => number; beatStarts: number[]; track?: TrackFile; toSource?: (frame: number) => number | null};
+const All = createContext<Ctx>({tweaks: {}, beatAt: () => 0, beatStarts: [0]});
+const Here = createContext<Tweak>({});
+
+/** beatStarts: the first frame of each beat, in order. */
+export const TweaksProvider: React.FC<{tweaks?: Tweaks; beatStarts: number[]; track?: TrackFile; toSource?: (frame: number) => number | null; children: React.ReactNode}> = ({
+  tweaks = {},
+  beatStarts,
+  track,
+  toSource,
+  children,
+}) => {
+  const beatAt = (frame: number) => {
+    for (let i = beatStarts.length - 1; i >= 0; i--) if (frame >= beatStarts[i]) return i;
+    return 0;
+  };
+  return <All.Provider value={{tweaks, beatAt, beatStarts, track, toSource}}>{children}</All.Provider>;
+};
+
+/** The tweak of the beat at any frame — for things placed by frame number, like sound cues. */
+export const useTweakAt = () => {
+  const {tweaks, beatAt} = useContext(All);
+  return (frame: number): Tweak => tweaks[String(beatAt(frame))] ?? {};
+};
+
+/** The tweak of the beat being drawn, inside a <Tweaked>. */
+export const useHereTweak = () => useContext(Here);
+
+/**
+ * Applies the current beat's size, position and timing to what is inside it. `move={false}` only hands the
+ * tweak down (see the rules above).
+ */
+export const Tweaked: React.FC<{move?: boolean; children: React.ReactNode}> = ({move = true, children}) => {
+  const f = useCurrentFrame();
+  const ctx = useContext(All);
+  const t = useTweakAt()(f);
+  // Follow: the beat keeps its layout but moves, scales and partly tilts with the head, measured from the
+  // first frame of the beat where the head is visible and tracked (a beat may open on a full-screen card).
+  // Untracked frames leave it exactly where it was designed.
+  let follow: React.CSSProperties | null = null;
+  if (move && t.follow && ctx.track && ctx.toSource) {
+    const track = ctx.track;
+    const src = ctx.toSource(f);
+    const beat = ctx.beatAt(f);
+    let src0: number | null = null;
+    for (let g = ctx.beatStarts[beat] ?? 0; g <= f && src0 === null; g++) {
+      const s = ctx.toSource(g);
+      if (s !== null && followTransform(track, s, s)) src0 = s;
+    }
+    follow = src !== null && src0 !== null ? followTransform(track, src, src0) : null;
+  }
+  const placed = (
+    <Here.Provider value={t}>
+      {move && (t.scale || t.dy) ? <AbsoluteFill style={{transform: `translateY(${t.dy ?? 0}px) scale(${t.scale ?? 1})`, transformOrigin: '50% 55%'}}>{children}</AbsoluteFill> : children}
+    </Here.Provider>
+  );
+  const inner = follow ? <AbsoluteFill style={follow}>{placed}</AbsoluteFill> : placed;
+  // Later (+) or earlier (−): everything inside runs on a clock offset by that many frames.
+  return move && t.shift ? (
+    <Sequence from={t.shift} layout="none">
+      {inner}
+    </Sequence>
+  ) : (
+    inner
+  );
+};