creator-editing-studio 1.3.6 → 1.3.8

package/template/CLAUDE.md

@@ -33,6 +33,14 @@ out/               finished renders.
 - `npm run ramp -- <clip> --ramp "3-5:1->0.4"` — eased speed ramps
 - `npm run sync -- <a> <b>` — line up a second angle or a separate mic
 - `npm run refstyle -- <reference clip>` — measure a reference edit's pacing and colour
+- `npm run refmotion -- <reference clip>` — study HOW a reference moves and sounds: every move's speed-up and
+  settle, the holds between moves, every cut, every frame across each transition (strips to look at),
+  standout sounds and how far they lead the picture, plus a spectrogram of the sound. Run it on any clip
+  the person shares as a reference, then LOOK at the strips and spectrogram before planning — numbers
+  alone do not show how one scene turns into the next.
+- `python scripts/track_head.py <clip> <first> <last> <out.json> --check <img>` — motion tracking: where the head
+  is, frame by frame (only when asked; see "Motion tracking" below)
+- `python scripts/setup_models.py --language <code>` — (re)install the models for a video language
 - `npm run stills -- <Comp> --every=45` — many stills at once, with contact sheets
 - `npm run check` — typecheck plus motion lint. Must pass.
 - `npm run motion-check -- <Comp> --scale=2` — frame-by-frame POP / CUT / STUTTER detection
@@ -83,6 +91,11 @@ that is when people can actually name what they like.
 Then transcribe with word-level timings. **The transcript is the truth, not the script and not
 what they remember saying.**
 
+The language comes from setup (`studio.config.json`; the transcriber and aligner read it). For Indian
+languages and Hinglish, **ask for the script or captions file first** and match it to the audio
+(`python scripts/align-script.py <media> <script.srt> --whole`) — matching known words beats recognising
+them, especially in code-mixed speech. Recognise only when there is no script.
+
 ### 2. The first cut, before the plan
 
 Cut the silence, the filler and the retakes first. This is mechanical — no design decisions in
@@ -116,6 +129,35 @@ what stops motion graphics drifting into generic.
 Then stop and ask: *here is the plan, change anything that is off, say go ahead and I will cut
 it.* Wait for a yes. "Go ahead", "yes", "approved", "ship it" is a yes. Silence is not.
 
+### 4. The live preview — after the yes, before any render
+
+Once they approve, build the edit and **open the live preview yourself** — `preview_edit({video})`. They must
+never have to ask for it, and never get a draft MP4 instead of it. It plays the edit in the chat with instant
+controls (size, position, timing, sound, icon, follow head) that cost no messages; see `preview/README.md` for
+what each video needs (`preview.tsx`, `TweaksProvider`).
+
+- A change typed in the preview arrives as a chat message: change only that beat, then call `preview_edit`
+  again so the updated preview opens below. **Undo** = `preview_undo`, then `preview_edit`.
+- **Save** and **Render** arrive as one message each, carrying their adjustments as JSON for the video's
+  `tweaks.json` — write it exactly.
+- Render only when they press **Render final video**, and give the estimated render time only then.
+- If the preview tool is missing (not switched on yet), say so in one line and fall back to rendering with
+  the time estimate. Never send them off to fix it mid-edit.
+- **"What's new" / "show me what's new"** → `preview_edit({video: "whats-new"})`: a short demo with one new
+  effect per beat, named on screen. Open it after every update too. It is a sandbox: let them press every
+  button. Save works as usual; if they press Render, say in one line that it is the demo, and that the same
+  buttons appear on every video of their own.
+
+### Motion tracking — only when asked
+
+Text or graphics can ride with the speaker's head. **Never a default** — use it when they ask ("make the title
+follow my head") or press **Follow head** in the preview. Track only the stretch that needs it:
+`python scripts/track_head.py <clip> <first source frame> <last source frame> src/videos/<video>/head-track.json
+--check work/<video>/track-check.jpg` (≈1 minute per 5 seconds), and **look at the check image** — the box must
+sit on the face in every tile — before using it. Then either pin a graphic with `<HeadPin>`
+(`src/components/HeadPin.tsx`) or turn on `follow` for the beat. A wide line will not fit above a head: split
+it. It takes half the head's tilt and trails two frames — attached, not glued.
+
 ### Keep it proportionate
 
 The plan is there to save them a rebuild, not to put a form in front of them.

package/template/package.json

@@ -17,6 +17,7 @@
     "stills": "node scripts/stills.mjs",
     "refs": "node scripts/refs.mjs",
     "refstyle": "node scripts/refstyle.mjs",
+    "refmotion": "node scripts/refmotion.mjs",
     "beats": "node scripts/beats.mjs",
     "sync": "node scripts/sync-audio.mjs",
     "ramp": "node scripts/ramp.mjs",
@@ -25,16 +26,20 @@
     "sound": "node scripts/sounddesign.mjs"
   },
   "dependencies": {
-    "ffmpeg-static": "5.3.0",
-    "ffprobe-static": "3.1.0",
+    "@modelcontextprotocol/ext-apps": "1.7.5",
+    "@modelcontextprotocol/sdk": "1.32.1",
     "@remotion/cli": "4.0.525",
     "@remotion/fonts": "4.0.525",
     "@remotion/google-fonts": "4.0.525",
     "@remotion/media-utils": "4.0.525",
     "@remotion/motion-blur": "4.0.525",
+    "esbuild": "0.28.1",
+    "ffmpeg-static": "5.3.0",
+    "ffprobe-static": "3.1.0",
     "react": "19.3.0",
     "react-dom": "19.3.0",
-    "remotion": "4.0.525"
+    "remotion": "4.0.525",
+    "zod": "4.5.4"
   },
   "devDependencies": {
     "@types/react": "19.3.0",

package/template/preview/README.md

@@ -0,0 +1,34 @@
+# Live preview
+
+The edit plays inside the chat, from its code. Nothing is rendered until you press **Render final video**.
+
+- **The video** on the left; **every beat** on the right with its timestamp.
+- **Instant controls under each beat** change the video on the spot — no chat, no waiting:
+  Size − / +, Move ↑ ↓, Timing Earlier / Later, Sound − / + / Mute, Icon, **Follow head** (the text rides
+  with the speaker's head — motion tracking), Reset.
+- **Save** writes all your adjustments in one go; **Render final video** saves them and renders, and tells
+  you how long the render will take first.
+- **"Something else? Describe it for Claude"** under a beat is for new ideas. It goes into the chat, Claude
+  makes the change, and the updated preview opens below. 📎 attaches a reference image (the app puts it in
+  your chat box — press Send).
+- **Undo last change** puts the edit back exactly as it was before the last change.
+
+Claude opens the preview by itself once you approve a plan. You never need to ask for it.
+
+## How it is switched on
+
+Setup registers it once (Claude Code: `claude mcp add live-preview --scope user -- node "<studio>/preview/server.mjs"`;
+Codex: `codex mcp add live-preview -- node "<studio>/preview/server.mjs"`). Apps load these when a chat starts,
+so it works from the next chat after setup.
+
+## For Claude
+
+- Open it with `preview_edit({video: "<folder in src/videos>"})` right after the plan is approved, and again
+  after every change you make. `preview_undo` restores the previous version.
+- Each video needs `src/videos/<video>/preview.tsx` exporting `Comp` (the edit + its voice as
+  `<Audio src={staticFile('__stems/voice.mp3')} />`, taking `tweaks` and passing them to the edit), `META`,
+  `BEATS` (one per plan row: name, time, beat, says, from, to, icons?, follow?) and `TWEAKS` (saved tweaks,
+  the tweaks.json path, icons, limits, trackFile?). The edit applies tweaks with `TweaksProvider` / `Tweaked`
+  from `src/lib/tweaks.tsx` — full-screen cards that hide cut joins never go inside a moving `Tweaked`.
+- The player cannot call this server or the computer (the app blocks it): everything it shows is packed into
+  the page when it opens, and it talks only by sending chat messages.

package/template/preview/lib.mjs

@@ -0,0 +1,180 @@
+/**
+ * What the live preview needs from a video, worked out once and used by both halves:
+ *   server.mjs   — the local preview server (our machine, reads files directly)
+ *   publish.mjs  — sends the same things to the online preview, so it works in any chat with no restart
+ *
+ * For a video it finds: the preview module (Comp, META, BEATS), every file the edit reads with
+ * staticFile() as a small preview copy, and the edit compiled to one script.
+ */
+import {spawnSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
+import {createRequire} from 'node:module';
+import {existsSync, mkdirSync, readdirSync, readFileSync, statSync} from 'node:fs';
+import {dirname, extname, join, resolve} from 'node:path';
+import {fileURLToPath, pathToFileURL} from 'node:url';
+
+export const HERE = dirname(fileURLToPath(import.meta.url));
+export const STUDIO = resolve(HERE, '..');
+export const PUBLIC = join(STUDIO, 'assets');
+export const SRC = join(STUDIO, 'src');
+const CACHE = join(HERE, '.cache');
+mkdirSync(CACHE, {recursive: true});
+
+const require = createRequire(join(STUDIO, 'package.json'));
+const esbuild = require('esbuild');
+const {FFMPEG} = await import(pathToFileURL(join(STUDIO, 'scripts', 'lib', 'media.mjs')).href);
+
+/* ----------------------------------------------------------------------------- which video */
+
+/**
+ * Everything the preview needs to know about one video, read from its own folder:
+ * src/videos/<name>/preview.tsx, the *Edit.tsx beside it, the asset folders its code mentions, its stems.
+ */
+export function configFor(name) {
+  if (!/^[a-z0-9][a-z0-9-]*$/.test(name ?? '')) throw new Error(`"${name}" is not a video name (lowercase letters, digits, dashes)`);
+  const code = join(SRC, 'videos', name);
+  if (!existsSync(join(code, 'preview.tsx'))) {
+    throw new Error(
+      `No src/videos/${name}/preview.tsx. Write it first: export Comp (the edit + its voice as <Audio src={staticFile('__stems/voice.mp3')} />), META and BEATS — copy src/videos/noufal/preview.tsx.`,
+    );
+  }
+  const source = readdirSync(code)
+    .filter((f) => /\.(tsx?|json)$/.test(f))
+    .map((f) => readFileSync(join(code, f), 'utf8'))
+    // The brand's own font files are named in tokens.ts, not in the video — without this the preview of
+    // every video fell back to a system font.
+    .concat(existsSync(join(SRC, 'design', 'tokens.ts')) ? [readFileSync(join(SRC, 'design', 'tokens.ts'), 'utf8')] : [])
+    .join('\n');
+  const folders = new Set(['fonts']);
+  // Only the fonts this edit names — the studio has many, and each one is ~1 MB in the page.
+  const fonts = [...new Set([...source.matchAll(/fonts\/([\w.-]+\.(?:ttf|otf|woff2?))/g)].map((m) => m[1]))];
+  // The whole folder path up to the file name — cutouts live a level deeper (cutouts/<shot>/f123.png).
+  for (const m of source.matchAll(new RegExp(`projects/${name}/((?:[\\w-]+/)+)`, 'g'))) folders.add(`projects/${name}/${m[1].replace(/\/$/, '')}`);
+  for (const f of folders) if (!existsSync(join(PUBLIC, f))) folders.delete(f);
+
+  const stems = {};
+  if (existsSync(join(STUDIO, 'work', name, 'voice.wav'))) stems.voice = `work/${name}/voice.wav`;
+  const out = join(STUDIO, 'out', name);
+  if (existsSync(out))
+    for (const f of readdirSync(out)) {
+      const m = f.match(/^stem-(\w+)\.wav$/);
+      if (m) stems[m[1]] = `out/${name}/${f}`;
+    }
+
+  // The edit is the file named *Edit.tsx; its composition id is that name (as registered in Root.tsx).
+  const editFile = readdirSync(code).find((f) => /Edit\.tsx$/.test(f));
+  return {
+    project: name,
+    preview: `src/videos/${name}/preview.tsx`,
+    edit: `src/videos/${name}/${editFile ?? 'preview.tsx'}`,
+    composition: editFile?.replace(/\.tsx$/, '') ?? null,
+    assets: [...folders],
+    fonts,
+    stems,
+  };
+}
+
+/* ----------------------------------------------------------------------------- the footage */
+
+/** Every file the edit asks for by name (staticFile), plus the stems. Keys are the names the edit uses. */
+function assetList(c) {
+  const files = {};
+  const addDir = (rel) => {
+    const dir = join(PUBLIC, rel);
+    if (!existsSync(dir)) return;
+    for (const f of readdirSync(dir)) {
+      if (rel === 'fonts' && c.fonts && !c.fonts.includes(f)) continue;
+      if (statSync(join(dir, f)).isFile()) files[`${rel}/${f}`] = join(dir, f);
+    }
+  };
+  for (const d of c.assets) addDir(d);
+  for (const [k, f] of Object.entries(c.stems ?? {})) if (existsSync(join(STUDIO, f))) files[`__stems/${k}.mp3`] = join(STUDIO, f);
+  return Object.fromEntries(Object.entries(files).filter(([k]) => /\.(mp4|mov|png|jpe?g|webp|svg|ttf|woff2?|mp3)$/i.test(k)));
+}
+
+/**
+ * Preview copies: small enough to travel inside the player page, good enough to judge the edit in a chat-
+ * sized player. Anything fits a 540px box on its longest side. The final render always uses the originals.
+ */
+const FIT = "scale='if(gt(iw,ih),min(540,iw),-2)':'if(gt(iw,ih),-2,min(540,ih))'";
+const PROXY = {
+  video: ['-vf', FIT, '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '31', '-g', '15', '-pix_fmt', 'yuv420p', '-movflags', '+faststart'],
+  image: ['-vf', FIT, '-c:v', 'libwebp', '-quality', '72'],
+  audio: ['-vn', '-ac', '2', '-ar', '44100', '-c:a', 'libmp3lame', '-b:a', '128k'],
+};
+const MIME = {'.mp4': 'video/mp4', '.webp': 'image/webp', '.mp3': 'audio/mpeg', '.svg': 'image/svg+xml', '.ttf': 'font/ttf', '.woff': 'font/woff', '.woff2': 'font/woff2', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png'};
+
+function proxyOf(name, original) {
+  const st = statSync(original);
+  // The settings are part of the key, so changing them makes new copies instead of reusing old ones.
+  const key = createHash('sha1').update(`${original}:${st.size}:${st.mtimeMs}:${JSON.stringify(PROXY)}`).digest('hex').slice(0, 16);
+  const ext = extname(name).toLowerCase();
+  const kind = /\.(mp4|mov)$/.test(ext) ? 'video' : /\.(png|jpe?g)$/.test(ext) ? 'image' : ext === '.mp3' ? 'audio' : null;
+  if (!kind) return {file: original, mime: MIME[ext] ?? 'application/octet-stream'};
+  const outExt = kind === 'video' ? '.mp4' : kind === 'image' ? '.webp' : '.mp3';
+  const out = join(CACHE, `${key}${outExt}`);
+  if (!existsSync(out)) {
+    const r = spawnSync(FFMPEG, ['-y', '-v', 'error', '-i', original, ...PROXY[kind], out], {encoding: 'utf8'});
+    if (r.status !== 0) throw new Error(`could not make a preview copy of ${name}: ${r.stderr}`);
+  }
+  return {file: out, mime: MIME[outExt]};
+}
+
+/** [{name, file, mime, size, hash}] — hash is of the preview copy, so unchanged files are never re-sent. */
+export function buildManifest(c) {
+  return Object.entries(assetList(c)).map(([name, original]) => {
+    const p = proxyOf(name, original);
+    const hash = createHash('sha256').update(readFileSync(p.file)).digest('hex').slice(0, 32);
+    return {name, file: p.file, mime: p.mime, size: statSync(p.file).size, hash};
+  });
+}
+
+/* ----------------------------------------------------------------------------- the edit, compiled */
+
+/**
+ * The edit runs inside the player, so it must use the player's own React and Remotion — two copies
+ * would break every hook. Those imports become reads of window.__shared.
+ */
+const SHARED = ['react', 'react-dom', 'react/jsx-runtime', 'react/jsx-dev-runtime', 'remotion'];
+const sharedPlugin = {
+  name: 'shared',
+  setup(b) {
+    b.onResolve({filter: new RegExp(`^(${SHARED.map((s) => s.replace('/', '\\/')).join('|')})$`)}, (a) => ({path: a.path, namespace: 'shared'}));
+    b.onLoad({filter: /.*/, namespace: 'shared'}, (a) => {
+      const keys = Object.keys(require(a.path)).filter((k) => /^[A-Za-z_$][\w$]*$/.test(k) && k !== 'default');
+      return {
+        contents: `const m = window.__shared[${JSON.stringify(a.path)}];\nexport default m;\n${keys.map((k) => `export const ${k} = m.${k};`).join('\n')}`,
+        loader: 'js',
+      };
+    });
+  },
+};
+
+const common = {bundle: true, jsx: 'automatic', logLevel: 'silent', loader: {'.png': 'dataurl', '.svg': 'dataurl'}};
+
+/** The edit as one script that sets window.__lpComp. Throws with a readable message if it doesn't compile. */
+export async function compileEdit(c) {
+  try {
+    const r = await esbuild.build({...common, entryPoints: [join(STUDIO, c.preview)], format: 'iife', globalName: '__lpComp', write: false, plugins: [sharedPlugin], define: {'process.env.NODE_ENV': '"development"'}});
+    return r.outputFiles[0].text;
+  } catch (e) {
+    throw new Error((e.errors ?? []).map((x) => `${x.location?.file ?? ''}:${x.location?.line ?? ''} ${x.text}`).join('\n') || String(e));
+  }
+}
+
+/**
+ * The player page. Customers get it pre-built (player.html, made when the studio ships), so they need none of
+ * the player's build tools; in the source studio it is built from ui/app.tsx.
+ */
+export async function playerHtml() {
+  const built = join(HERE, 'player.html');
+  if (existsSync(built) && !existsSync(join(HERE, 'ui', 'app.tsx'))) return readFileSync(built, 'utf8');
+  return buildUi();
+}
+
+export async function buildUi() {
+  const r = await esbuild.build({...common, entryPoints: [join(HERE, 'ui', 'app.tsx')], format: 'iife', write: false, minify: true, define: {'process.env.NODE_ENV': '"production"'}});
+  const js = r.outputFiles[0].text;
+  return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
+<body><div id="root"></div><script>${js.replace(/<\/script>/g, '<\\/script>')}</script></body></html>`;
+}